{
 "G_med": 2.573387642438244,
 "G_med_api_basis_P_vs_A": -1.1922125858968342,
 "G_med_flag_gt10": false,
 "additions": [
  {
   "baseline_categories_sha256": "55c86278fb2ca0e8407f58eeb5338406f298a1bca426efd2ae7b52dad2752f6f",
   "baseline_systems_sha256": "6255c1c1014f148ad59eb43f78ded416d3dbb6604ebd6af3d530935edabc8f6b",
   "date": "2026-10-07",
   "keys": [
    "kev-27b",
    "pplx-decider-v1-1-27b",
    "decider-chat-gemma4-31b",
    "jpt-35b-a3b",
    "jebadiah-27b",
    "simplejev-qwen3.8-27b-selfhost",
    "bespoke-nimble-9b-v3",
    "gev-26b-decide",
    "torchcast-decision-27b",
    "decision-2-0-vega-27b",
    "jade",
    "jev-27b-autotrust"
   ],
   "label": "a6",
   "note": "Twelve open-weights systems from the top 20 of the community Jev Decision Index on Hugging Face, measured on the same sealed v1.6 pool, scorer and cost basis as every other row.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "9d512dc13d7d876b233560e8710ed5f64d83f555be90141fa62644f22d5c9421"
  },
  {
   "baseline_categories_sha256": "a2fe49209522d498b3be81e884f9751392c9c135669f11fbb5b8bdda0b2765ce",
   "baseline_systems_sha256": "2ff7130b5dc1946f005044b773c703835d902b808dec9a7f1556f2b9452b2657",
   "date": "2026-10-07",
   "keys": [
    "surogate-rune-26b-a4b-v3-v16",
    "xor-26b-a4b"
   ],
   "label": "a7",
   "note": "Surogate Rune 26B-A4B v3 and Xor 26B-A4B from the top 20 of the community Jev Decision Index on Hugging Face, measured on the same sealed v1.6 pool, scorer and cost basis as every other row.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "9d512dc13d7d876b233560e8710ed5f64d83f555be90141fa62644f22d5c9421"
  },
  {
   "baseline_categories_sha256": "1922c9d5f5391b2e280288f4b5414edf24ac82e06b06cb6fb8bda3d34bf6342e",
   "baseline_systems_sha256": "67d9aca6921a4f148b1137b2fcd438e582463b6e09a1e9904bf00a60c0b82812",
   "date": "2026-10-07",
   "keys": [
    "h2o-lightning-4b",
    "kahn1"
   ],
   "label": "a8",
   "note": "H2O-Lightning-4B v1.1 (H2O.ai) and Kahn1 4B (Okura66), two Qwen3.5-4B fine-tunes submitted as JevBench add-requests, measured on the same sealed v1.6 pool, scorer and cost basis as every other row; both are priced with labelled cost estimates (Kahn1's server reports no token usage).",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "7a5b4d4c7b7ce966c1d7048d5106d50a6a700c8d89372e223ac276f41540be9d"
  },
  {
   "baseline_categories_sha256": "07d69bc9f0638886a19c5d63f2473f9b2677b66950b7d2ea79f0d7c7902f4d69",
   "baseline_systems_sha256": "20b282fdf8597453965eb1bb6fe2d2fd47fbd07963adb5c9da97df862c309318",
   "date": "2026-10-08",
   "keys": [
    "aplomb-1",
    "apus-openjev-v1-35b-a3b",
    "apus-openjev-v1-9b",
    "bb-qwen3.5-4b-lora",
    "classone-gemma4-e2b",
    "classone-qwen3.5-9b",
    "coco-decision-4b",
    "decisio-gemma-4-12b-v080",
    "decisio-gemma-4-12b-v090",
    "diffusion-jev",
    "evalengine-decision-4b",
    "exaone-4.0-1.2b-jev-v0.3",
    "hopper-12b-trained",
    "janus-4b",
    "jul-fast",
    "messier-one-v0.2",
    "seb-9b",
    "spx-cd-omni",
    "tacet-sonata",
    "typecastlm-1.4.0",
    "watersheep",
    "watt-flash-0.1"
   ],
   "label": "a11",
   "note": "Completed current-pool submissions added; historical versions and self-hosted/API conditions are disclosed.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "4d2833926c2daf284ce98e1fea6d9223dfc08341905778eb77391734638c27d2"
  },
  {
   "baseline_categories_sha256": "25dddf8fab1b6e240bbe68429f5986f74ef6407ee935adc99972390458c05263",
   "baseline_systems_sha256": "e3b67483db81438b8ec1c8adfab8dc8b12d9ef0ddec32617b0dec1d59766fbb2",
   "date": "2026-10-08",
   "keys": [
    "bobcat-flash-1.2",
    "decider-12b",
    "decider-12b-v1",
    "decisio-gemma-4-31b-v080",
    "mercury-decide",
    "rene-1-31b-fp8",
    "spx-cd-flash",
    "spx-cd-pro"
   ],
   "label": "a12",
   "note": "Approval-required completed candidates; current open/API top-five impact reviewed separately.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "4d2833926c2daf284ce98e1fea6d9223dfc08341905778eb77391734638c27d2"
  }
 ],
 "amendments": [
  {
   "date": "2026-10-05",
   "revision": "v1.6.1",
   "text": "5 Oct 2026, v1.6.1 (Florian's decision): hosted/API systems now answer the same full item set as self-hosted systems (1,200 sealed + 300 public). Their earlier answers on the API subsets were reused and only the missing sealed items were sent; every item was logged in the exposure ledger before it was sent. API rows are no longer equated. Consequence: every v1.6.0 sealed item has now been seen by at least two providers, so the whole v1.6.0 sealed draw is retired for future releases; v1.6.2 onward draws fresh items from the reserve."
  },
  {
   "date": "2026-10-05",
   "revision": "v1.6.1",
   "text": "5 Oct 2026, v1.6.1 cost rule (Florian's decision): cost per 1,000 decisions is now measured on one common item set for every system: all items except the 23 very long items that lie outside the Jev reference's accepted input range. Before, a system that processed those items paid for their tokens while a system that refused them did not. Refusals still count against Intelligence exactly as before. Hosted APIs with a per-token tariff are priced from their measured tokens on this common set; carried per-decision prices of self-hosted systems are unchanged. Effect: Sage 1.3.0 USD 0.0247 per 1,000 decisions on the common set (0.0650 on all 1,500 items); Jev 1.13.0 0.0323 (unchanged)."
  }
 ],
 "benchmark": "JevBench",
 "board": {
  "A": {
   "leader_wording": null,
   "markers": [],
   "order": [
    "singularity-super-decider",
    "kettei-fat",
    "fastino-glide-api",
    "torchcast-decision-4b-v1.3",
    "liquid-d1-ultra",
    "subq-snap-1.1",
    "drex-max",
    "nimble-v4-max",
    "singularity-super-decider-thinking",
    "subq-snap-preview",
    "h2o-lightning-4b",
    "sage-1.3.0",
    "kettei-chub",
    "h2o-lightning-31b",
    "torchcast-decision-4b-v1.2",
    "liquid-d1",
    "mercury-decide",
    "decisio-gemma-4-31b-v080",
    "basal-omni-1.0-ultra",
    "jev-1.13.0",
    "quyet-1-0-large",
    "celeris-1-decision",
    "torchcast-decision-4b-v1.1",
    "launchables-myles-g-31b",
    "arcbright-elf-1-31b",
    "decider-12b",
    "xevn-4b",
    "jev-1.14.0",
    "torchcast-decision-12b",
    "decider-12b-v1",
    "metask_jev_rain_31b_trained",
    "vllm-sr-d3-mini",
    "bobcat-1.3",
    "microsoft-decision-1",
    "wity-1",
    "winnow-12b",
    "deck31b",
    "jeff_1_0_large",
    "cygnet",
    "launchables-myles-g-12b",
    "wald-26b-a4b",
    "decisio-gemma-4-12b-v080",
    "attest-31b-v1.0",
    "sage-1.4.0",
    "decisio-gemma-4-12b-v090",
    "jev-omni",
    "metask_jev_rain_12b_trained",
    "bobcat-flash-1.2",
    "xor-26b-a4b",
    "spx-cd-flash",
    "messier-one-v0.4",
    "decider-chat-gemma4-31b",
    "hopper-12b-trained",
    "metask_jev_rain_31b_trained_v3",
    "surogate-rune-26b-a4b-v3-v16",
    "gev-26b-decide",
    "spx-cd-omni",
    "metask-jev-rain-4b",
    "shisa-de-1",
    "openai-decisions",
    "instinct",
    "blink-v0-3-26b-a4b-nvfp4",
    "hyland-hs1",
    "elephant-1",
    "diffusion-jev",
    "rene-1-31b-fp8",
    "wald-4b-v2-1",
    "pplx-decider-flash-v1.2",
    "bespoke-nimble-9b-v3",
    "aplomb-1",
    "respan-span-02",
    "seb-9b-v2",
    "apus-openjev-v1-9b",
    "arcbright-elf-1-12b",
    "jeffan-31b-1710",
    "plumb-4b",
    "spx-cd-pro",
    "messier-one-v0.2",
    "spark-s1-4b-v6",
    "knowline-31b-gen1",
    "swanone",
    "ajev-gemma4-31b-lm6q",
    "quyet-1-0-medium",
    "nimble-v4",
    "vansa-3.4",
    "jpt-4b",
    "lev",
    "ninfer-qwen3.8-flash-next",
    "decider-4b-v2",
    "mammoth-1",
    "jeffan-4b-2010",
    "jev-local",
    "gpt-6-luna-low",
    "kushal-gemma4-31b-it-autoloops",
    "wald-4b-v2",
    "gpt-6-luna",
    "classone-qwen3.5-9b",
    "jevk5-v0.3-4b",
    "metask-jev-4b",
    "deck-4b-v1-0",
    "kahn1",
    "clef-flash",
    "evalengine-decision-4b",
    "decisor_4b",
    "janus-4b",
    "qwen35-9b-jev-data-mix-v2",
    "seb-9b",
    "jevk5-v02",
    "tern-35b-a3b",
    "amos-1-medium",
    "aleria-judge-1.1.1",
    "jevone",
    "fastino-glide",
    "jebadiah-9b-v2-1",
    "verdin-4b",
    "typecastlm-1.4.0",
    "blink-4b",
    "jiwo-4b",
    "imajev-4b-rtx5090-a2",
    "simplejev-qwen3.6-35b-a3b",
    "uniphore-4b-decider",
    "decision2-lux-9b",
    "hopper",
    "malkuth-4b",
    "jqv",
    "jpt-35b-a3b",
    "decider-35b-a3b",
    "coco-decision-4b",
    "manchego-v21-v15",
    "decision-4b-v12",
    "standard_one_3b_sh_v4",
    "fastino-glide-1.2-open",
    "raw-qwen3-4b-instruct-2507",
    "startlux-decision-v1.1-27b-instant",
    "standard_one_3b_sh",
    "rsi-jev-v6.1-vl-27b",
    "gpt-5.6-luna",
    "jev-27b-autotrust",
    "torchcast-decision-27b",
    "nimble-9b",
    "nimble-v4-max-open",
    "pplx-decider-v1.2",
    "pplx-decider-v1-1-27b",
    "vllm-sr-d3-turbo",
    "vllm-sr-d3",
    "reflex-4b",
    "vera-large",
    "amos-1-large",
    "jade",
    "nagi-enormous",
    "typecastlm",
    "jebadiah-27b",
    "iota-decide-4b",
    "decision-2-0-vega-27b",
    "jebadiah-9b-v2",
    "standard_one_8b_sh",
    "decision-4b-v11",
    "autojev-27b",
    "pjev-2b",
    "eikos-27b",
    "instinct-dual-4b",
    "ninfer-qwen3.8-27b",
    "compass-0.2.0",
    "jeb-35b-a3b",
    "openjev-thinking",
    "open-jev-zefan-9b",
    "gemini-3.1-flash-lite",
    "standardone-8b",
    "clef",
    "arsenelupin-v1.1",
    "system-one-open",
    "jev-qwen3.5-9b-base-nvfp4",
    "goddragon-arbiter-4b",
    "roudone-decider-4b",
    "classone-gemma4-e4b",
    "semif-qwen3.5-4b",
    "openjev-sglang",
    "decisionnode-edge-0.9.2",
    "ryotide_qwen9",
    "classone-qwen3.5-4b",
    "jez-large-1.0.0",
    "ryotide-qwen",
    "chon-1.0-12b",
    "noma",
    "knowline-35b-a3b-gen1.5",
    "bev-bonsai-27b",
    "litjev",
    "localjev-qwen3.5-4b",
    "system-one-sg",
    "startlux-decision-v1.1-27b-autothink",
    "kev-8b",
    "decision-2b",
    "opensourcejev-qwen35-4b-q4km",
    "imajev-9b",
    "open-alternative-jev",
    "raw-qwen3-8b",
    "liquid-d1-3b",
    "krino-2b",
    "imajev-2b",
    "decider-2b",
    "nemotron-diffusion-8b",
    "deepseek-flash",
    "kev-4b",
    "clef-omni",
    "kev-27b",
    "lumma-fev-27b",
    "decisionnode-edge-mini-0.9.2",
    "jiwo-0.8b",
    "malkuth-2b",
    "qwen3-reranker-4b",
    "simplejev-qwen3.8-27b-selfhost",
    "zerank-2",
    "exaone-4.0-1.2b-jev-v0.3",
    "open-jev-zefan-2b",
    "simplejev-qwen3.8-27b",
    "vegaml-4b",
    "open-jev-zefan-27b-v1.1",
    "raw-phi-4-mini",
    "decision-machine-1",
    "simplejev-qwen3.5-0.8b",
    "jevact",
    "smalljev",
    "gutsy-0.8b-v0.3",
    "vega-u-qwen3.5-4b",
    "purplemist-mini-1.0",
    "kev-0.6b",
    "jul-fast",
    "opendecision",
    "classone-gemma4-e2b",
    "fastino-gliner-2-5-decide",
    "deem-0.8-v1",
    "decision-fast",
    "quyet-1-0-small-en",
    "classone-qwen3.5-2b",
    "laya-typed-decisions",
    "vega-u-qwen3.5-0.8b",
    "watersheep",
    "openjev-verdict-1.4",
    "raw-qwen3-0.6b",
    "lev-350m",
    "raw-qwen3-1.7b",
    "kev-0.5b",
    "jez-small-1.0.0",
    "openjev-verdict",
    "quyet-1-0-small",
    "watt-flash-0.1",
    "quyet-1-0-tiny",
    "phocinae-largha-150m-v1",
    "liquid-d1-omni-600m",
    "vegaml-0.8b",
    "open-jev-deberta-v3-large",
    "julia-1",
    "verdict-small",
    "tacet-sonata",
    "laya",
    "mxbai-rerank-base-v2",
    "laya-multilingual",
    "certo",
    "clm-8b",
    "bge-reranker-v2-m3",
    "gte-reranker-modernbert-base",
    "mirror",
    "open-jev-json-canvas-joshuasp",
    "qwen3.8-27b"
   ]
  },
  "B": {
   "leader_wording": null,
   "markers": [],
   "order": [
    "singularity-super-decider",
    "kettei-fat",
    "fastino-glide-api",
    "drex-max",
    "subq-snap-1.1",
    "singularity-super-decider-thinking",
    "liquid-d1-ultra",
    "h2o-lightning-31b",
    "torchcast-decision-4b-v1.3",
    "subq-snap-preview",
    "nimble-v4-max",
    "basal-omni-1.0-ultra",
    "sage-1.3.0",
    "quyet-1-0-large",
    "decisio-gemma-4-31b-v080",
    "launchables-myles-g-31b",
    "mercury-decide",
    "kettei-chub",
    "arcbright-elf-1-31b",
    "h2o-lightning-4b",
    "liquid-d1",
    "torchcast-decision-4b-v1.2",
    "metask_jev_rain_31b_trained",
    "jev-1.13.0",
    "deck31b",
    "bobcat-1.3",
    "jeff_1_0_large",
    "wity-1",
    "decider-12b",
    "celeris-1-decision",
    "jev-1.14.0",
    "torchcast-decision-12b",
    "decider-12b-v1",
    "wald-26b-a4b",
    "attest-31b-v1.0",
    "metask_jev_rain_12b_trained",
    "sage-1.4.0",
    "winnow-12b",
    "bobcat-flash-1.2",
    "torchcast-decision-4b-v1.1",
    "metask_jev_rain_31b_trained_v3",
    "microsoft-decision-1",
    "launchables-myles-g-12b",
    "xevn-4b",
    "cygnet",
    "spx-cd-flash",
    "jev-omni",
    "vllm-sr-d3-mini",
    "xor-26b-a4b",
    "decider-chat-gemma4-31b",
    "decisio-gemma-4-12b-v080",
    "decisio-gemma-4-12b-v090",
    "surogate-rune-26b-a4b-v3-v16",
    "messier-one-v0.4",
    "hopper-12b-trained",
    "gev-26b-decide",
    "openai-decisions",
    "metask-jev-rain-4b",
    "shisa-de-1",
    "elephant-1",
    "spx-cd-omni",
    "hyland-hs1",
    "diffusion-jev",
    "blink-v0-3-26b-a4b-nvfp4",
    "instinct",
    "rene-1-31b-fp8",
    "bespoke-nimble-9b-v3",
    "wald-4b-v2-1",
    "respan-span-02",
    "pplx-decider-flash-v1.2",
    "jeffan-31b-1710",
    "aplomb-1",
    "spx-cd-pro",
    "apus-openjev-v1-9b",
    "seb-9b-v2",
    "knowline-31b-gen1",
    "arcbright-elf-1-12b",
    "ajev-gemma4-31b-lm6q",
    "plumb-4b",
    "gpt-6-luna-low",
    "swanone",
    "mammoth-1",
    "gpt-6-luna",
    "spark-s1-4b-v6",
    "messier-one-v0.2",
    "ninfer-qwen3.8-flash-next",
    "kushal-gemma4-31b-it-autoloops",
    "quyet-1-0-medium",
    "nimble-v4",
    "vansa-3.4",
    "lev",
    "jev-local",
    "jpt-4b",
    "decider-4b-v2",
    "jeffan-4b-2010",
    "wald-4b-v2",
    "classone-qwen3.5-9b",
    "clef-flash",
    "kahn1",
    "metask-jev-4b",
    "deck-4b-v1-0",
    "jevk5-v0.3-4b",
    "decisor_4b",
    "aleria-judge-1.1.1",
    "evalengine-decision-4b",
    "qwen35-9b-jev-data-mix-v2",
    "fastino-glide",
    "janus-4b",
    "tern-35b-a3b",
    "jevone",
    "seb-9b",
    "amos-1-medium",
    "jevk5-v02",
    "simplejev-qwen3.6-35b-a3b",
    "jebadiah-9b-v2-1",
    "verdin-4b",
    "typecastlm-1.4.0",
    "blink-4b",
    "jiwo-4b",
    "imajev-4b-rtx5090-a2",
    "jpt-35b-a3b",
    "decision2-lux-9b",
    "decider-35b-a3b",
    "uniphore-4b-decider",
    "fastino-glide-1.2-open",
    "gpt-5.6-luna",
    "hopper",
    "malkuth-4b",
    "startlux-decision-v1.1-27b-instant",
    "jqv",
    "rsi-jev-v6.1-vl-27b",
    "torchcast-decision-27b",
    "standard_one_3b_sh_v4",
    "pplx-decider-v1-1-27b",
    "nimble-v4-max-open",
    "pplx-decider-v1.2",
    "vllm-sr-d3-turbo",
    "jev-27b-autotrust",
    "vllm-sr-d3",
    "coco-decision-4b",
    "manchego-v21-v15",
    "decision-4b-v12",
    "vera-large",
    "amos-1-large",
    "nagi-enormous",
    "raw-qwen3-4b-instruct-2507",
    "nimble-9b",
    "jade",
    "jebadiah-27b",
    "decision-2-0-vega-27b",
    "standard_one_3b_sh",
    "eikos-27b",
    "autojev-27b",
    "openjev-thinking",
    "reflex-4b",
    "ninfer-qwen3.8-27b",
    "gemini-3.1-flash-lite",
    "clef",
    "standard_one_8b_sh",
    "typecastlm",
    "open-jev-zefan-9b",
    "jebadiah-9b-v2",
    "iota-decide-4b",
    "decision-4b-v11",
    "jeb-35b-a3b",
    "pjev-2b",
    "compass-0.2.0",
    "standardone-8b",
    "instinct-dual-4b",
    "openjev-sglang",
    "arsenelupin-v1.1",
    "system-one-open",
    "jev-qwen3.5-9b-base-nvfp4",
    "classone-gemma4-e4b",
    "goddragon-arbiter-4b",
    "roudone-decider-4b",
    "semif-qwen3.5-4b",
    "knowline-35b-a3b-gen1.5",
    "decisionnode-edge-0.9.2",
    "ryotide_qwen9",
    "classone-qwen3.5-4b",
    "bev-bonsai-27b",
    "startlux-decision-v1.1-27b-autothink",
    "jez-large-1.0.0",
    "chon-1.0-12b",
    "litjev",
    "ryotide-qwen",
    "noma",
    "system-one-sg",
    "localjev-qwen3.5-4b",
    "kev-8b",
    "imajev-9b",
    "opensourcejev-qwen35-4b-q4km",
    "raw-qwen3-8b",
    "decision-2b",
    "open-alternative-jev",
    "deepseek-flash",
    "liquid-d1-3b",
    "krino-2b",
    "imajev-2b",
    "decider-2b",
    "kev-27b",
    "nemotron-diffusion-8b",
    "clef-omni",
    "lumma-fev-27b",
    "kev-4b",
    "decisionnode-edge-mini-0.9.2",
    "simplejev-qwen3.8-27b-selfhost",
    "simplejev-qwen3.8-27b",
    "jiwo-0.8b",
    "open-jev-zefan-27b-v1.1",
    "malkuth-2b",
    "qwen3-reranker-4b",
    "open-jev-zefan-2b",
    "zerank-2",
    "exaone-4.0-1.2b-jev-v0.3",
    "vegaml-4b",
    "raw-phi-4-mini",
    "decision-machine-1",
    "simplejev-qwen3.5-0.8b",
    "jevact",
    "smalljev",
    "gutsy-0.8b-v0.3",
    "vega-u-qwen3.5-4b",
    "purplemist-mini-1.0",
    "kev-0.6b",
    "classone-gemma4-e2b",
    "jul-fast",
    "opendecision",
    "fastino-gliner-2-5-decide",
    "deem-0.8-v1",
    "decision-fast",
    "quyet-1-0-small-en",
    "classone-qwen3.5-2b",
    "laya-typed-decisions",
    "vega-u-qwen3.5-0.8b",
    "watersheep",
    "raw-qwen3-0.6b",
    "openjev-verdict-1.4",
    "raw-qwen3-1.7b",
    "lev-350m",
    "kev-0.5b",
    "jez-small-1.0.0",
    "openjev-verdict",
    "quyet-1-0-small",
    "watt-flash-0.1",
    "quyet-1-0-tiny",
    "phocinae-largha-150m-v1",
    "liquid-d1-omni-600m",
    "vegaml-0.8b",
    "open-jev-deberta-v3-large",
    "julia-1",
    "verdict-small",
    "tacet-sonata",
    "laya",
    "mxbai-rerank-base-v2",
    "laya-multilingual",
    "certo",
    "clm-8b",
    "bge-reranker-v2-m3",
    "gte-reranker-modernbert-base",
    "mirror",
    "open-jev-json-canvas-joshuasp",
    "qwen3.8-27b"
   ]
  },
  "C": {
   "leader_wording": null,
   "markers": [],
   "order": [
    "singularity-super-decider",
    "kettei-fat",
    "fastino-glide-api",
    "torchcast-decision-4b-v1.3",
    "liquid-d1-ultra",
    "subq-snap-1.1",
    "drex-max",
    "nimble-v4-max",
    "singularity-super-decider-thinking",
    "subq-snap-preview",
    "h2o-lightning-4b",
    "sage-1.3.0",
    "kettei-chub",
    "h2o-lightning-31b",
    "torchcast-decision-4b-v1.2",
    "liquid-d1",
    "mercury-decide",
    "decisio-gemma-4-31b-v080",
    "basal-omni-1.0-ultra",
    "jev-1.13.0",
    "quyet-1-0-large",
    "launchables-myles-g-31b",
    "arcbright-elf-1-31b",
    "decider-12b",
    "jev-1.14.0",
    "torchcast-decision-12b",
    "metask_jev_rain_31b_trained",
    "bobcat-1.3",
    "wity-1",
    "decider-12b-v1",
    "deck31b",
    "jeff_1_0_large",
    "celeris-1-decision",
    "wald-26b-a4b",
    "attest-31b-v1.0",
    "winnow-12b",
    "sage-1.4.0",
    "metask_jev_rain_12b_trained",
    "bobcat-flash-1.2",
    "metask_jev_rain_31b_trained_v3",
    "spx-cd-flash",
    "decider-chat-gemma4-31b",
    "microsoft-decision-1",
    "launchables-myles-g-12b",
    "xor-26b-a4b",
    "jev-omni",
    "torchcast-decision-4b-v1.1",
    "cygnet",
    "surogate-rune-26b-a4b-v3-v16",
    "openai-decisions",
    "rene-1-31b-fp8",
    "xevn-4b",
    "elephant-1",
    "decisio-gemma-4-12b-v080",
    "respan-span-02",
    "decisio-gemma-4-12b-v090",
    "vllm-sr-d3-mini",
    "messier-one-v0.4",
    "diffusion-jev",
    "jeffan-31b-1710",
    "hopper-12b-trained",
    "spx-cd-pro",
    "bespoke-nimble-9b-v3",
    "hyland-hs1",
    "metask-jev-rain-4b",
    "gev-26b-decide",
    "knowline-31b-gen1",
    "spx-cd-omni",
    "ajev-gemma4-31b-lm6q",
    "shisa-de-1",
    "instinct",
    "blink-v0-3-26b-a4b-nvfp4",
    "mammoth-1",
    "wald-4b-v2-1",
    "gpt-6-luna-low",
    "gpt-6-luna",
    "swanone",
    "kushal-gemma4-31b-it-autoloops",
    "pplx-decider-flash-v1.2",
    "aplomb-1",
    "seb-9b-v2",
    "apus-openjev-v1-9b",
    "arcbright-elf-1-12b",
    "plumb-4b",
    "messier-one-v0.2",
    "aleria-judge-1.1.1",
    "spark-s1-4b-v6",
    "fastino-glide",
    "quyet-1-0-medium",
    "nimble-v4",
    "vansa-3.4",
    "jpt-4b",
    "lev",
    "ninfer-qwen3.8-flash-next",
    "decider-4b-v2",
    "jeffan-4b-2010",
    "jev-local",
    "wald-4b-v2",
    "classone-qwen3.5-9b",
    "jevk5-v0.3-4b",
    "metask-jev-4b",
    "deck-4b-v1-0",
    "kahn1",
    "clef-flash",
    "evalengine-decision-4b",
    "decisor_4b",
    "janus-4b",
    "simplejev-qwen3.6-35b-a3b",
    "qwen35-9b-jev-data-mix-v2",
    "seb-9b",
    "jevk5-v02",
    "tern-35b-a3b",
    "fastino-glide-1.2-open",
    "amos-1-medium",
    "jevone",
    "startlux-decision-v1.1-27b-instant",
    "rsi-jev-v6.1-vl-27b",
    "gpt-5.6-luna",
    "torchcast-decision-27b",
    "jebadiah-9b-v2-1",
    "verdin-4b",
    "typecastlm-1.4.0",
    "nimble-v4-max-open",
    "pplx-decider-v1.2",
    "pplx-decider-v1-1-27b",
    "blink-4b",
    "vllm-sr-d3-turbo",
    "vllm-sr-d3",
    "vera-large",
    "amos-1-large",
    "jiwo-4b",
    "imajev-4b-rtx5090-a2",
    "nagi-enormous",
    "jev-27b-autotrust",
    "decision-2-0-vega-27b",
    "uniphore-4b-decider",
    "decision2-lux-9b",
    "jpt-35b-a3b",
    "hopper",
    "malkuth-4b",
    "eikos-27b",
    "jebadiah-27b",
    "jqv",
    "openjev-thinking",
    "decider-35b-a3b",
    "coco-decision-4b",
    "manchego-v21-v15",
    "decision-4b-v12",
    "standard_one_3b_sh_v4",
    "clef",
    "gemini-3.1-flash-lite",
    "autojev-27b",
    "raw-qwen3-4b-instruct-2507",
    "standard_one_3b_sh",
    "nimble-9b",
    "reflex-4b",
    "jade",
    "typecastlm",
    "ninfer-qwen3.8-27b",
    "iota-decide-4b",
    "jebadiah-9b-v2",
    "standard_one_8b_sh",
    "decision-4b-v11",
    "pjev-2b",
    "instinct-dual-4b",
    "knowline-35b-a3b-gen1.5",
    "compass-0.2.0",
    "jeb-35b-a3b",
    "open-jev-zefan-9b",
    "standardone-8b",
    "arsenelupin-v1.1",
    "system-one-open",
    "jev-qwen3.5-9b-base-nvfp4",
    "goddragon-arbiter-4b",
    "roudone-decider-4b",
    "classone-gemma4-e4b",
    "startlux-decision-v1.1-27b-autothink",
    "semif-qwen3.5-4b",
    "openjev-sglang",
    "decisionnode-edge-0.9.2",
    "ryotide_qwen9",
    "classone-qwen3.5-4b",
    "jez-large-1.0.0",
    "ryotide-qwen",
    "chon-1.0-12b",
    "noma",
    "bev-bonsai-27b",
    "litjev",
    "localjev-qwen3.5-4b",
    "system-one-sg",
    "kev-8b",
    "decision-2b",
    "opensourcejev-qwen35-4b-q4km",
    "deepseek-flash",
    "imajev-9b",
    "open-alternative-jev",
    "raw-qwen3-8b",
    "liquid-d1-3b",
    "krino-2b",
    "kev-27b",
    "imajev-2b",
    "decider-2b",
    "nemotron-diffusion-8b",
    "kev-4b",
    "lumma-fev-27b",
    "clef-omni",
    "decisionnode-edge-mini-0.9.2",
    "simplejev-qwen3.8-27b-selfhost",
    "jiwo-0.8b",
    "malkuth-2b",
    "qwen3-reranker-4b",
    "simplejev-qwen3.8-27b",
    "zerank-2",
    "exaone-4.0-1.2b-jev-v0.3",
    "open-jev-zefan-2b",
    "open-jev-zefan-27b-v1.1",
    "vegaml-4b",
    "raw-phi-4-mini",
    "decision-machine-1",
    "simplejev-qwen3.5-0.8b",
    "jevact",
    "smalljev",
    "gutsy-0.8b-v0.3",
    "vega-u-qwen3.5-4b",
    "purplemist-mini-1.0",
    "kev-0.6b",
    "jul-fast",
    "opendecision",
    "classone-gemma4-e2b",
    "fastino-gliner-2-5-decide",
    "deem-0.8-v1",
    "decision-fast",
    "quyet-1-0-small-en",
    "classone-qwen3.5-2b",
    "laya-typed-decisions",
    "vega-u-qwen3.5-0.8b",
    "watersheep",
    "openjev-verdict-1.4",
    "raw-qwen3-0.6b",
    "lev-350m",
    "raw-qwen3-1.7b",
    "kev-0.5b",
    "jez-small-1.0.0",
    "openjev-verdict",
    "quyet-1-0-small",
    "watt-flash-0.1",
    "quyet-1-0-tiny",
    "phocinae-largha-150m-v1",
    "liquid-d1-omni-600m",
    "vegaml-0.8b",
    "open-jev-deberta-v3-large",
    "julia-1",
    "verdict-small",
    "tacet-sonata",
    "laya",
    "mxbai-rerank-base-v2",
    "laya-multilingual",
    "certo",
    "clm-8b",
    "bge-reranker-v2-m3",
    "gte-reranker-modernbert-base",
    "mirror",
    "open-jev-json-canvas-joshuasp",
    "qwen3.8-27b"
   ]
  }
 },
 "bootstrap": {
  "B": 1000,
  "bootstrap_seed": 16,
  "g_med_fixed": true
 },
 "headline": "A",
 "label": "JevBench v1.7.0: 10-second limit per decision; every row re-scored from its recorded per-decision times",
 "measurement_notes": [
  "Method v1.7: from v1.7.0 a decision that takes longer than 10 seconds (the measured time, strictly more than 10.000 s) counts as a wrong answer and as zero calibration for that item (each item type's calibration is multiplied by the share of its decisions answered within 10 s). Every row is re-scored from its recorded per-decision times: Intelligence, Calibration, Capability, the composites and their intervals are recomputed with the same frozen scorer plus the rule; the Speed axis now uses all decisions given (it replaces the 124-item subset of v1.6). Cost, tokens, the draws, the raw outputs and the field median are unchanged; no model was run again for the re-scored rows. Every re-scored row shows its v1.6 values next to the new ones.",
  "Time limit on measured time: the limit uses the measured latency, not the displayed estimate (x2 + 0.15 s for self-hosted rows), so a self-hosted row can show an adjusted p95 above 10 s with no decision over the limit. In total 1,455 decisions across 59 rows are over the limit.",
  "Re-measure pending: 5 rows (chon-1.0-e4b, nev-2b, modernbert-jev-v2, mghafiri-qwen3.5-0.8b-decision-model, reflex-instinct-0.6b) recorded their per-decision times on a loaded CPU host, so they cannot be scored under the time limit and no estimate is published. They stay on the board with their v1.6 values, labelled as v1.6, unranked, until they are measured again.",
  "Re-measured in v1.7.0: 4 rows that were pending (loaded CPU host) were measured again on the same items on a dedicated GPU. Each GPU row replaces the earlier CPU row in full, scores and times; GPU times are never combined with CPU scores, and the CPU values stay visible as the row's v1.6 values. pjev-2b (Kailune AI, frozen MiniCPM5-2B + merged LoRA r64, letter-logit readout, T=1 stub) (pjev-2b): no decision took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw. PurpleMIST-Mini-1.0 (Olaverse) (purplemist-mini-1.0): no decision took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw. Vega-U (Qwen3.5-4B reader) (vega-u-qwen3.5-4b): 15 of its 1,500 decisions took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw. Verdin-4B (Victor Bennett, Qwen3.5-4B + merged LoRA r16 + pointer head) (verdin-4b): no decision took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw.",
  "Open/sealed gap penalty: the public-versus-sealed gap penalty is computed from the row's own results, so a decision that now fails can slightly reduce that penalty and raise Intelligence a little. This affects 1 row in this release (shisa-de-1 (shisa-ai, Gemma-4-26B-A4B): +0.006 Intelligence points); it is a side effect of the existing penalty, not a change to it.",
  "Dated carry: 14 rows measured before v1.6 (v1.5 method) are carried as published and not re-scored.",
  "Equated API rows: the pool re-runs behind the A4 and A5 equating offsets are scored under the same rule, so the offsets moved with them. These rows keep the 2-decimal precision of their earlier publication.",
  "New in v1.7.0: 26 rows measured for the first time under the 10 s rule. AJev lm6q (Gemma 4 31B) (ajev-gemma4-31b-lm6q): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Aleria Judge 1.1.1 (ALERIA-ML, inference technique on stock Gemma 4 31B, author server, self-hosted) (aleria-judge-1.1.1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Arcbright Elf 1 31B (arcbright-elf-1-31b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Bobcat 1.3 (Gemma 4 31B) (bobcat-1.3): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. DecisionNode Edge 0.9.2 (decisionnode-edge-0.9.2): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. DecisionNode Edge Mini 0.9.2 (decisionnode-edge-mini-0.9.2): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. baselabs elephant-1 (9B) (elephant-1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Fastino GLiDE 1.2 (open weights, self-hosted) (fastino-glide-1.2-open): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Jeffan-31b-1710 (jeffan-31b-1710): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Jeffan-4B-2010 (jeffan-4b-2010): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Jez-large-1.0.0 (pragmaticfrontier, Gemma-4-E4B label-logit decision model, author server, self-hosted) (jez-large-1.0.0): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Jez-small-1.0.0 (pragmaticfrontier, Qwen3.5-0.8B label-logit decision model, author server, self-hosted) (jez-small-1.0.0): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. KnowLine 31B-Gen1 (knowline-31b-gen1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. KnowLine 35B-A3B-Gen1.5 (knowline-35b-a3b-gen1.5): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. FrontiersMind Lumma-fev-27B (lumma-fev-27b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. baselabs mammoth-1 (27B) (mammoth-1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Nimble-v4-max (open weights, self-hosted) (nimble-v4-max-open): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 1 of its 1,500 decision took longer than 10 seconds and is scored by the rule. Its mandatory disclosure and licence note are shown on the row. Seb-9B-v2 (ironbcc, Qwen3.5-9B fine-tune, one forward pass, stock vLLM, self-hosted) (seb-9b-v2): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Super Decider (singularity-super-decider): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 11 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Super Decider (thinking) (singularity-super-decider-thinking): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 4 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Standard One 3B SH v4 (Standard Thinking) (standard_one_3b_sh_v4): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Tern-35B-A3B (matthewagi, Qwen3.6 35B-A3B MoE decision model, author server, self-hosted) (tern-35b-a3b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Vera Large (vera-large): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. vLLM-SR d3-mini (vllm-sr-d3-mini): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Wald 26B-A4B (Gemma 4 26B-A4B) (wald-26b-a4b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Xevn-4B (xevn-4b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row.",
  "Board effect: the 10 s rule (method v1.7) re-scored every row; the merged-board top fives before/after v1.7.0 were compared over all 36 mixed, open-weights and API variants (including cost caps of 7x and 10x the reference cost, each also with no latency cap). The reference cost is that of Jev 1.13.0, USD 0.0323 per 1,000 decisions, and the official cap is 2x that. Together with 4 release-board lists, 37 of 40 top-five lists change. Top-five changes caused by: the 10 s rule (method v1.7) and the new rows arcbright-elf-1-31b, fastino-glide-1.2-open, knowline-31b-gen1, mammoth-1, singularity-super-decider, singularity-super-decider-thinking (measured for the first time in v1.7.0); entering a top five: arcbright-elf-1-31b, fastino-glide-1.2-open, kettei-fat, knowline-31b-gen1, mammoth-1, singularity-super-decider, singularity-super-decider-thinking; leaving a top five: chon-1.0-12b, drex-max, fastino-glide-api, h2o-lightning-31b, liquid-d1-ultra, nimble-v4-max, qwen3.8-27b, subq-snap-1.1, subq-snap-preview, vllm-sr-d3, vllm-sr-d3-turbo. ",
  "Publication of v1.7.0: Florian selected \"Publish (recommended)\" on Telegram card #18506 (2026-10-24T14:13:05Z), the top-five preview card with preview PNG sha256 b082a945643c70be13f1628cfb9a908f4d11f1acdc72e33b2785ffecad617d2e. Florian top-five preview approval: Telegram card #18506; selected \"Publish (recommended)\" (2026-10-24T14:13:05Z); preview PNG SHA-256 b082a945643c70be13f1628cfb9a908f4d11f1acdc72e33b2785ffecad617d2e."
 ],
 "method": {
  "decision_limit_s": 10,
  "label": "v1.7",
  "rule": "latency_s > 10.000 s -> failed",
  "treatment": "share"
 },
 "n_ranked": 264,
 "not_measured": [
  {
   "addendum": null,
   "author": "unknown",
   "display": "AutoJev-27B (RTX PRO 6000)",
   "key": "autojev-27b-rtxpro6000-a2",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/denis-pplx/autojev-27b",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Clause Logic",
   "display": "Bosun v3.1 0.6B",
   "key": "bosun-v31-0.6b",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/Hanno-Labs/bosun-v3.1-0.6b",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2 (Fastino, gliner2.5-base)",
   "key": "gliner2",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/fastino-ai/GLiNER2",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2 large (Fastino)",
   "key": "gliner2-large",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/fastino/gliner2-large-v1",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2.5 multi (Fastino, 287M)",
   "key": "gliner2.5-multi",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/fastino/gliner2.5-multi-v1",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2.5 small (Fastino, 74M)",
   "key": "gliner2.5-small",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/fastino/gliner2.5-small-v1",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Logan Markewich",
   "display": "jeff (Logan Markewich, GLiFormer 400M)",
   "key": "jeff",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/logan-markewich/jeff",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "MantisShrimpdev",
   "display": "Jobe Qwen3.5-4B (frozen)",
   "key": "jobe-qwen3.5-4b",
   "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.",
   "repo": "https://benchmarkheaven.com/jev-models/jobe-qwen3.5-4b",
   "status": "not measured"
  },
  {
   "addendum": {
    "id": "A1",
    "label": "v1.5 roster addendum A1",
    "release": "v1.5"
   },
   "author": "unknown",
   "display": "mica-v01-4b",
   "key": "mica-v01-4b",
   "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: The frozen refusal policy stopped the run after 1,088 of 1,624 rows: 27 documented refusals were mapped to HTTP 422, then three consecutive passthrough HTTP 400 refusals triggered exit 6. The remaining 536 rows have no scores, so this system is not eligible for an official rank.",
   "repo": "https://benchmarkheaven.com/jev-models/mica-v01-4b",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Cactus Compute",
   "display": "Needle 3 (Cactus, 2-bit, local CPU)",
   "key": "needle-3",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/cactus-compute/needle",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Cactus Compute",
   "display": "Needle 3, options as tools (post-hoc adapter mode)",
   "key": "needle-3-tools",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/cactus-compute/needle",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Igor L. / NInfer contributors",
   "display": "NInfer Qwen3.8-27B NVFP4 (T=1.5)",
   "key": "ninfer-qwen3.8-27b-t1.5",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/igorls/ninfer",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "razorback16 / Codiv",
   "display": "OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)",
   "key": "openjev-razorback16",
   "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.",
   "repo": "https://benchmarkheaven.com/jev-models/openjev-razorback16",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "kshetrajna12",
   "display": "reflex-27b (Qwen3.8-27B)",
   "key": "reflex-27b",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/kshetrajna12/reflex",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "unknown",
   "display": "Surogate Rune 26B-A4B v3 (RTX PRO 6000)",
   "key": "surogate-rune-26b-a4b-v3-rtxpro6000-a2",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/api/models/surogate/rune-26b-a4b-GGUF",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "wfzyx (Victor Hugo)",
   "display": "Von (wfzyx, Option-Marker 395M)",
   "key": "von-395m",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/wfzyx/von",
   "status": "not measured"
  }
 ],
 "noul_method": {
  "applied": "O1S",
  "decisive_source_sha256": "159554cb4ebc276bac6e24559b3f326a47398b97479d0048647411ed64322e49"
 },
 "options": {
  "A": {
   "intelligence_floor": 50,
   "weights": {
    "calibration": 25,
    "cost": 25,
    "intelligence": 25,
    "speed": 25
   }
  },
  "B": {
   "intelligence_floor": 50,
   "weights": {
    "calibration": 20,
    "cost": 20,
    "intelligence": 40,
    "speed": 20
   }
  },
  "C": {
   "intelligence_floor": 60,
   "weights": {
    "calibration": 25,
    "cost": 25,
    "intelligence": 25,
    "speed": 25
   }
  }
 },
 "overnight": {
  "a2_note": "History: hosted APIs were first measured on API subsets (A, then A2 and A3) and equated to the self-hosted scale in v1.6.0. From v1.6.1 they answer the same full set as self-hosted systems, so no equating is applied; the earlier subsets A, A2 and A3 are retired.",
  "exposure": {
   "classifier-dev-fast": {
    "display": "classifier-dev-fast (classifier.dev)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "decision-machine-1": {
    "display": "decision-machine-1 (milliseconds.ai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "deepseek-flash": {
    "display": "deepseek-flash (deepseek)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "fastino-gliner-2-5-decide": {
    "display": "fastino-gliner-2-5-decide (fastino)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "gemini-3.1-flash-lite": {
    "display": "gemini-3.1-flash-lite (google)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "gpt-5.6-luna": {
    "display": "gpt-5.6-luna (openai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "gpt-6-luna": {
    "display": "gpt-6-luna (openai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "gpt-6-luna-low": {
    "display": "gpt-6-luna-low (openai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "instinct": {
    "display": "instinct (zoowork)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "instinct-dual-4b": {
    "display": "instinct-dual-4b (zoowork)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "jev-1.13.0": {
    "display": "jev-1.13.0 (typesafe)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "jevact": {
    "display": "jevact (jevact)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "kushal-gemma4-31b-it-autoloops": {
    "display": "kushal-gemma4-31b-it-autoloops (autoloops)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "openjev-sglang": {
    "display": "openjev-sglang (modal via modal)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "qwen3.8-27b": {
    "display": "qwen3.8-27b (chutes)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "sage-1.3.0": {
    "display": "Sage 1.3.0 (Levanto Labs)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "simplejev-qwen3.6-35b-a3b": {
    "display": "simplejev-qwen3.6-35b-a3b (featherless)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "simplejev-qwen3.8-27b": {
    "display": "simplejev-qwen3.8-27b (featherless)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "system-one-open": {
    "display": "system-one-open (modal via modal)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "vansa-3.4": {
    "display": "vansa-3.4 (vansa)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "wity-1-always": {
    "display": "wity-1-always (wity via railway)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "wity-1-auto": {
    "display": "wity-1-auto (wity via railway)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "wity-1-off": {
    "display": "wity-1-off (wity via railway)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   }
  },
  "notes": [
   "Scores are the official v1.6.0 scorer (score_v16.py, method option B / O1S, bootstrap B = 1,000) over complete outputs only; each scoring round is kept separately.",
   "Hosted APIs are scored on the same full item set as self-hosted systems (S 1,200 + P 300, v1.6.1) and are not equated.",
   "GPU-class deviations: reproducible recipes used the hardware listed per row, including H100 for large fast-lane decoders and RTX 6000 for the baseline; hardware differences remain in measured latency. The standard x2 + 0.15 s self-hosted adjustment is an assumption, not a hardware normalization.",
   "Jev-class caps use a fixed reference: Jev 1.13.0 as measured in v1.5 (p50 0.62 s, USD 0.0323 per 1k answers); caps = 2x (1.23 s, USD 0.0646). Jev's own v1.6 p50 is 0.24 s. Wity auto remains outside the latency cap and stays ranked in Composite A. OFF and ALWAYS are unranked variants of the AUTO main row.",
   "6 further measured candidates await a separate publication decision.",
   "Sage 1.3.0 (Levanto Labs) was measured on 5 Oct from Sandy (Helsinki), text only. Cost uses the Levanto list tariff (USD 0.05/M input, USD 10/M output; levanto.ai/pricing, read 5 Oct) over all 600 answered rows: USD 0.0766/1,000 decisions. It exceeds the frozen Jev-class cost cap and is excluded from the Capability headline.",
   "A fresh Monday Jev 1.13.0 re-check on A3 measured p50 0.239 s and equated Capability 77.5 versus the official 76.5, within the confidence interval; the official v1.6.0 Jev row is retained."
  ],
  "round": "score-v161-3 (v1.6.1)",
  "scored_utc": "2026-10-06 00:39:36 UTC"
 },
 "protocol": "jevbench::v1.7",
 "provisional": false,
 "restates": {
  "board_release": "v1.6.43",
  "feed_sha256": "f1006546d7d9614132df86542176127835b1be5eb39344bc074ac9a83f2dbb0e"
 },
 "revision": "v1.7.0",
 "revision_history": [
  {
   "date": "2026-10-05",
   "revision": "v1.6.1",
   "summary": "Hosted APIs (fastino-gliner-2-5-decide, jev-1.13.0, sage-1.3.0, wity-1, wity-1-always, wity-1-off) answer the full 1,500-item set instead of the 600-item API subset and are no longer equated; self-hosted rows unchanged; whole v1.6.0 sealed draw retired. Cost per 1,000 decisions is measured on one common item set (all items except the 23 outside the Jev reference input range) for every token-priced row."
  },
  {
   "date": "2026-10-05",
   "revision": "v1.6.0",
   "summary": "Rotating item draw (1,200 sealed + 300 public); hosted APIs on API subsets equated to the self-hosted scale; /jev-models/v1.6.0 stays available unchanged."
  }
 ],
 "roster_count": 285,
 "run_kind": "full-rescore",
 "sample": {
  "open": 300,
  "published_open": 300,
  "sealed": 1200,
  "total": 1500
 },
 "sealed_share_of_intelligence": 0.5,
 "source_note": "285 rows: the 259 rows of the live board at v1.6.43 plus 26 new rows measured under the 10 s rule. 250 are re-scored under the 10 s rule from their recorded per-decision times (score_v17 sha256 6363458d..., share treatment, B=1000, the published per-row seeds), 5 are re-measure pending and shown as v1.6. 4 are re-measured on the same items on a GPU and replace their CPU rows in full. No model was run again for the re-scored rows.",
 "source_sha256": "e15347783b8bedeeddd9017a831608a957f5bf0386f05902c50161ce0468f911",
 "status": "published",
 "systems": [
  {
   "addendum": null,
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Levanto Labs",
   "axes": {
    "calibration": 91.59817606608073,
    "cost": 58.231203787388296,
    "intelligence": 65.59290944525489,
    "speed": 91.59383325786439
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.034436035156249975,
      "ece_score": 93.11279296875,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 93.11279296875,
      "score_before_time_limit": 93.11279296875,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.16533333333333333,
      "brier": 0.08087898763020833,
      "ece": 0.049375,
      "n": 375,
      "score": 90.125,
      "score_before_time_limit": 90.125,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.968,
      "interval90_width": 0.2808888888888889,
      "n": 375,
      "nrps": 0.06392779541015624,
      "score": 91.55673522949218,
      "score_before_time_limit": 91.55673522949218,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05246875
     }
    },
    "score": 91.59817606608073
   },
   "capability": 78.59554275566781,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     71.4032274840039,
     74.84235710953574
    ],
    "B": [
     68.73566282385947,
     73.77740634012592
    ],
    "C": [
     71.0269206102275,
     74.84235710953574
    ]
   },
   "cost": {
    "basis": "Levanto list tariff, top tier: USD 0.05 per 1M input tokens, USD 10 per 1M output tokens (https://levanto.ai/pricing, read 5 Oct 2026)",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.06497606666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.024677048070413
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Sage 1.3.0 (Levanto Labs)",
   "endpoint_condition": "Levanto hosted API, text only, measured from Sandy (Helsinki) on 5 Oct 2026; full set 1,200 sealed + P300 (v1.6.1); not evaluated on ImageJevBench.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 65.67742817331194,
    "I_sealed": 65.50839071719783,
    "base": 65.59290944525489,
    "excess": -2.4043501863241326,
    "gap": 0.16903745611411125,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 85.05619838199217,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 83.01352626612143,
       "judge": 84.17930125247197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 54.291235334713605,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 57.68485080323005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.80397727272727,
       "hard": 52.14120370370371,
       "judge": 68.19385593220339,
       "standard": 45.94907407407408
      }
     },
     "sealed|choice": {
      "cc": 79.68397614698543,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 82.70637794693668,
       "hard": 73.02036199095022,
       "judge": 85.1024208566108,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 56.36040792532405,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 41.747572815533985,
       "judge": 57.89473684210527,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 60.480788079284,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 76.96974734042553,
       "hard": 51.94546568627452,
       "judge": 64.89053143712576,
       "standard": 62.692338197969534
      }
     }
    }
   },
   "jevbench_score": 73.72797965992304,
   "key": "sage-1.3.0",
   "last_measured_on": "2026-10-05",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "levanto-sage-v1.3 (Sage 1.3.0)",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9488817891373802,
    "decisive_rate": 0.8346666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 8,
    "axes": {
     "calibration": 91.59817606608073,
     "cost": 58.231203787388296,
     "intelligence": 65.59290944525489,
     "speed": 93.61638181239104
    },
    "board_release": "v1.6.43",
    "capability": 78.59554275566781,
    "capability_api_board_rank": 9,
    "capability_open_board_rank": null,
    "capability_rank": 17,
    "composite_ci95": {
     "A": [
      71.70514550946575,
      75.17412681523324
     ],
     "B": [
      68.95926995267655,
      74.03508105576502
     ],
     "C": [
      71.32573829825638,
      75.17412681523324
     ]
    },
    "jevbench_score": 74.04992179175468,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 9,
    "ranks": {
     "A": 9,
     "B": 10,
     "C": 9,
     "capability": 33
    },
    "scores": {
     "A": 74.04992179175468,
     "B": 72.1884444725083,
     "C": 74.04992179175468
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.15245595201849937,
     "p50_s_raw": 0.15245595201849937,
     "p95_s_adjusted": 0.2852446019649505,
     "p95_s_raw": 0.2852446019649505
    },
    "views": {
     "Balanced 33:33:33": 69.60497929786021,
     "Emphasis on Accuracy 60:20:20": 67.94266119076805,
     "Emphasis on Cost 20:20:60": 64.56093721161619,
     "Emphasis on Speed 20:60:20": 77.56249365186707,
     "Intelligence only": 65.59290944525489
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 12,
   "ranked": true,
   "ranks": {
    "A": 12,
    "B": 13,
    "C": 12,
    "capability": 38
   },
   "repo": "https://levanto.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.72797965992304,
    "B": 71.94344239717388,
    "C": 73.72797965992304
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.17092861235141754,
    "p50_s_raw": 0.17092861235141754,
    "p95_s_adjusted": 0.4053236149251461,
    "p95_s_raw": 0.4053236149251461
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.55241460870812,
      92.09209592889854
     ],
     "I": [
      60.000783824692206,
      70.03012866456243
     ],
     "capability": [
      74.15252653755351,
      80.27098937533351
     ],
     "composite": {
      "A": [
       71.4032274840039,
       74.84235710953574
      ],
      "B": [
       68.73566282385947,
       73.77740634012592
      ],
      "C": [
       71.0269206102275,
       74.84235710953574
      ]
     },
     "seed": 114
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.59817606608073,
     "capability": 78.59554275566781,
     "intelligence": 65.59290944525489
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8493333333333334,
      "calibration": 93.11279296875,
      "cc": 80.73188522493676,
      "cc_by_type": {
       "choice": 80.73188522493676
      },
      "cc_tiered": 82.3700872644888,
      "cc_tiered_open": 85.05619838199217,
      "cc_tiered_sealed": 79.68397614698543,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 80.73188522493676
     },
     "noul": {
      "accuracy": 0.792,
      "calibration": 90.125,
      "cc": 58.400000000000006,
      "cc_by_type": {
       "noul": 58.400000000000006
      },
      "cc_tiered": 55.32582163001882,
      "cc_tiered_open": 54.291235334713605,
      "cc_tiered_sealed": 56.36040792532405,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 58.400000000000006
     },
     "score": {
      "accuracy": 0.68,
      "calibration": 91.55673522949218,
      "cc": 60.08634868421052,
      "cc_by_type": {
       "score": 60.08634868421052
      },
      "cc_tiered": 59.08281944125702,
      "cc_tiered_open": 57.68485080323005,
      "cc_tiered_sealed": 60.480788079284,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.08634868421052
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.22612697189255,
    "Emphasis on Accuracy 60:20:20": 67.72558799363198,
    "Emphasis on Cost 20:20:60": 64.36490392701504,
    "Emphasis on Speed 20:60:20": 76.72033411975153,
    "Intelligence only": 65.59290944525489
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Inception (API, free on OpenRouter) - Florian work order, 6 Oct 2026",
   "axes": {
    "calibration": 81.34341651168008,
    "cost": 62.08313746631354,
    "intelligence": 65.90701345473542,
    "speed": 84.81995518783543
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11045316563089375,
      "ece_score": 77.90936687382126,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 77.90936687382126,
      "score_before_time_limit": 77.90936687382126,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.10242587601078168,
      "brier": 0.10151182638394682,
      "ece": 0.08813961357701941,
      "n": 371,
      "score": 82.37207728459612,
      "score_before_time_limit": 82.37207728459612,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8753387533875339,
      "interval90_width": 0.1458897922312557,
      "n": 369,
      "nrps": 0.06239733540901056,
      "score": 83.74880537662287,
      "score_before_time_limit": 83.74880537662287,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.13131327852926614
     }
    },
    "score": 81.34341651168008
   },
   "capability": 73.62521498320774,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": [
     69.82880671245154,
     73.74963246907883
    ],
    "B": [
     67.70690015261091,
     72.8950112029009
    ],
    "C": [
     69.63070886080774,
     73.74963246907883
    ]
   },
   "cost": {
    "basis": "LABELLED ESTIMATE, not charged - and nothing WAS charged: usage.cost is 0 on every one of the 1,500 calls. Inception publishes NO paid tariff for Mercury Decide: OpenRouter lists it free-only (prompt 0, completion 0) and the non-free slug inception/mercury-decide has no endpoints at all (404 'No endpoints found', read 7 Oct 2026), so there is nothing bookable. Because the system is API-only with undisclosed weights, the open rows' size-class estimate table cannot apply either. The row is therefore priced from THE SAME OPERATOR'S cheapest published per-token tariff on the SAME platform, inception/mercury-2.5 at USD 0.04 per 1M input and USD 0.15 per 1M output, which IS a key in the frozen 25 Sep snapshot - x this row's own measured tokens. Output tokens are 3 per decision on average (4,434 over 1,477 answers) because the probability is read from the model rather than written out, so the output rate is nearly inert and the Cost axis is set by the input rate alone. RELEASE-LANE LINE, recorded and not acted on, and it is RUN 57'S SHAPE EXACTLY - one mapping line, not a new price: the rate inception/mercury-2.5 (0.04, 0.15) is already IN the dated snapshot; what is missing is only a base-model decision line pointing at it, so pricing_v15.reference() raises on every inception/* string. receipts/COST-ALTERNATIVES-R58.json has the two alternatives' arithmetic already done.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.01807942666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.018360961408259987
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Mercury Decide (Inception; System One decisions API, served free on OpenRouter as inception/mercury-decide:free)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), COMPLETE: 1,500 of 1,500 dispatched, 1,477 answered, 23 invalid. Measured through the release lane's reviewed run_api_full_v161.py UNCHANGED, on Florian's 5 Oct amendment (hosted systems answer the same full S u P as self-hosted ones); provider inception, proxy openrouter, every sealed item exposure-logged before the first request. Florian work order of 6 Oct 2026 ('put it on JevBench', under https://x.com/airesearch12/status/2107569862486659111). NO MAPPING DECISION OF OURS: Mercury Decide publishes the same /v1/systemone contract as Jev and all three primitives come back in the canonical shape (noul a float; choice a label plus probabilities keyed by the item's own option labels; score probabilities keyed by level index '0'..'n-1' - 0-based, unlike ClassOne's), so the vendored typesafe adapter's request body, answer mapping and validations are reused UNCHANGED. or_decisions_r58.OpenRouterDecisionsAdapter rewrites ONLY the URL, because OpenRouter serves the contract at POST /api/alpha/decisions rather than /v1/systemone - its own 400 from chat/completions names that path - and it does so by calling the parent run(), so none of the reviewed mapping is duplicated. The pinned harness/vendor tree was NOT edited; the module is reached by PYTHONPATH. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM: the context window is 32,768 tokens and all 23 came back as HTTP 422 with the provider's own 'upstream_error: Decision service could not complete the request' - a disclosed refusal, which the harness counts as the system's own rather than as an outage. The invalid set is EXACTLY those 23 and nothing else failed in 1,477 answers. RUN IN TWO PASSES BECAUSE OF A FREE-TIER BURST LIMIT, AND IT DOES NOT MOVE THE SPEED FIGURE: items 1-736 ran back to back at ~1.7 req/s until OpenRouter returned a header-less 429 ('rate_limit_exceeded: Request or spending limit exceeded'), which is run_v16's own stop rule; the row was resumed at 1 req/s and completed. The two passes' medians differ by 0.038 s (0.3624 s vs 0.3246 s over 1,477 answers; 0.3624 vs 0.3104 on the 124 items the Speed axis actually uses), and run_v16 sleeps the spacing BEFORE starting the timer, so the pacing is not in the measurement. Receipt LATENCY-SEGMENTS-mercury-R58.txt. Production API, so NO latency adjustment. Serial, one decision per request, no retry, no batching. add-requests run 58",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 67.12906498043914,
    "I_sealed": 64.68496192903172,
    "base": 65.90701345473542,
    "excess": -0.12928459103081957,
    "gap": 2.4441030514074242,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.71168908755723,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 81.12614029569046,
       "judge": 80.22412656558997,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 58.71014492753623,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 62.96536092622397,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.561695179434636,
       "hard": 63.165822850913635,
       "judge": 76.80814242854491,
       "standard": 42.0020976967579
      }
     },
     "sealed|choice": {
      "cc": 74.09139259954716,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 68.43891402714931,
       "judge": 79.51582867783985,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 57.27197132719671,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 35.92233009708738,
       "judge": 65.78947368421053,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 62.691521860351266,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.70067055435705,
       "hard": 51.597448628789465,
       "judge": 70.50916076191972,
       "standard": 69.64863562411932
      }
     }
    }
   },
   "jevbench_score": 72.24968682544169,
   "key": "mercury-decide",
   "last_measured_on": "2026-10-07",
   "licence": null,
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "provider-reported measurement model slug: inception/mercury-decide:free",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9039039039039038,
    "decisive_rate": 0.8975741239892183,
    "supported": true,
    "valid": 371
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 11,
    "axes": {
     "calibration": 81.34341651168008,
     "cost": 62.08313746631354,
     "intelligence": 65.90701345473542,
     "speed": 85.47752366478576
    },
    "board_release": "v1.6.43",
    "capability": 73.62521498320774,
    "capability_api_board_rank": 13,
    "capability_open_board_rank": null,
    "capability_rank": 27,
    "composite_ci95": {
     "A": [
      69.93954261051735,
      73.87316393178861
     ],
     "B": [
      67.79015712351674,
      72.99152541270097
     ],
     "C": [
      69.74081685972264,
      73.87316393178861
     ]
    },
    "jevbench_score": 72.36824049637728,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 15,
    "ranks": {
     "A": 15,
     "B": 15,
     "C": 15,
     "capability": 51
    },
    "scores": {
     "A": 72.36824049637728,
     "B": 70.97659528968215,
     "C": 72.36824049637728
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.33213604614138603,
     "p50_s_raw": 0.33213604614138603,
     "p95_s_adjusted": 0.8529658075422044,
     "p95_s_raw": 0.8529658075422044
    },
    "views": {
     "Balanced 33:33:33": 69.80103192778668,
     "Emphasis on Accuracy 60:20:20": 68.18948202584015,
     "Emphasis on Cost 20:20:60": 66.49451324072818,
     "Emphasis on Speed 20:60:20": 75.32698952826067,
     "Intelligence only": 65.90701345473542
    }
   },
   "provenance": {
    "carried_metadata_from": "Current-pool API measurement registry",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 17,
   "ranked": true,
   "ranks": {
    "A": 17,
    "B": 17,
    "C": 17,
    "capability": 62
   },
   "repo": "https://openrouter.ai/inception/mercury-decide:free",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.24968682544169,
    "B": 70.88533274889458,
    "C": 72.24968682544169
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.34386441484093666,
    "p50_s_raw": 0.34386441484093666,
    "p95_s_adjusted": 0.9585554625838959,
    "p95_s_raw": 0.9585554625838959
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.84031884046848,
      84.00969333082946
     ],
     "I": [
      60.19803361460591,
      70.00098427505237
     ],
     "capability": [
      69.08542722449891,
      76.48291495498401
     ],
     "composite": {
      "A": [
       69.82880671245154,
       73.74963246907883
      ],
      "B": [
       67.70690015261091,
       72.8950112029009
      ],
      "C": [
       69.63070886080774,
       73.74963246907883
      ]
     },
     "seed": 91
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.34341651168008,
     "capability": 73.62521498320774,
     "intelligence": 65.90701345473542
    },
    "per_type": {
     "choice": {
      "accuracy": 0.808,
      "calibration": 77.90936687382126,
      "cc": 75.44594223354774,
      "cc_by_type": {
       "choice": 75.44594223354774
      },
      "cc_tiered": 76.9015408435522,
      "cc_tiered_open": 79.71168908755723,
      "cc_tiered_sealed": 74.09139259954716,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.44594223354774
     },
     "noul": {
      "accuracy": 0.8026666666666666,
      "calibration": 82.37207728459612,
      "cc": 60.53333333333333,
      "cc_by_type": {
       "noul": 60.53333333333333
      },
      "cc_tiered": 57.99105812736647,
      "cc_tiered_open": 58.71014492753623,
      "cc_tiered_sealed": 57.27197132719671,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.53333333333333
     },
     "score": {
      "accuracy": 0.712,
      "calibration": 83.74880537662287,
      "cc": 62.21714605598172,
      "cc_by_type": {
       "score": 62.21714605598172
      },
      "cc_tiered": 62.82844139328762,
      "cc_tiered_open": 62.96536092622397,
      "cc_tiered_sealed": 62.691521860351266,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.21714605598172
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.65404525909453,
    "Emphasis on Accuracy 60:20:20": 68.1052419126018,
    "Emphasis on Cost 20:20:60": 66.41440648974917,
    "Emphasis on Speed 20:60:20": 75.01947411259273,
    "Intelligence only": 65.90701345473542
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amin Roudaki (@aminry)",
   "axes": {
    "calibration": 88.71627256952715,
    "cost": 51.66530928015878,
    "intelligence": 70.42321091828492,
    "speed": 91.13376683561437
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06150753988606317,
      "ece_score": 87.69849202278736,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.69849202278736,
      "score_before_time_limit": 87.69849202278736,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.06469002695417791,
      "brier": 0.04879523639868886,
      "ece": 0.04694201082318837,
      "n": 371,
      "score": 90.61159783536232,
      "score_before_time_limit": 90.61159783536232,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9349593495934959,
      "interval90_width": 0.1980578139114724,
      "n": 369,
      "nrps": 0.057879253805657324,
      "score": 87.83872785043177,
      "score_before_time_limit": 87.83872785043177,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09267309459285371
     }
    },
    "score": 88.71627256952715
   },
   "capability": 79.56974174390604,
   "class": "Jev-compatible decision readout on frozen base",
   "composite_ci95": {
    "A": [
     69.84990305266089,
     72.80044769282709
    ],
    "B": [
     68.87338684458163,
     73.02825429245821
    ],
    "C": [
     69.84990305266089,
     72.80044769282709
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff) and CLEAN: unlike run 54's FP8 sibling, the frozen pricing snapshot resolves this EXACT base - google/gemma-4-31B-it - to USD 0.09 per 1M input and USD 0.34 per 1M output. Read before the pod and asserted in make_registry_r55.py. x this row's own measured tokens, the deck31b method. Nothing is provisional and no release-lane line is open on it. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04022032666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04084664184157075
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisio v0.8.0 on gemma-4-31B-it (frozen, FP8 on load, one prefill per state, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/aminry/decisio tag v0.8.0 (5b42101a191d222062a795194b0aedb5003bf6ea) - the same tag whose other two entries run 54 measured - installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version, and served with their own `python -m decisio.serve.vllm_engine --base gemma-4-31b`. --base carries google/gemma-4-31B-it@842da379, the official weights, FROZEN: no fine-tuning, no adapter, no task registered; vLLM 0.30.0 quantizes it to FP8 when it loads (30.6 GiB). That profile's measured settings: temperature 4.672 for choice questions and 5.252 for the rest, the 12B entry's prompt unchanged, a system turn, the chat template's own answer position, every single-token form of each letter summed, no tokens generated. New in 0.8.0 on this base and the reason it is worth its own row: several questions of a state are scored in ONE pass after a single prefill. The author's own limit is a 32,768-token prompt, so the pool's 23 items at ~80,000 tokens are a documented capacity limit, not a defect. THE AUTHOR STATES ONE 96 GB CARD; this is one H100 80 GB (Lium) and no flag of theirs was changed to make it fit. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #169",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 67.59136348711468,
    "I_sealed": 73.25505834945518,
    "base": 70.42321091828492,
    "excess": -8.23708250477875,
    "gap": -5.663694862340506,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.82494232345373,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 64.40342781806196,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 73.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 58.8283751910304,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.01492747107596,
       "hard": 63.1937957087831,
       "judge": 57.45226814139626,
       "standard": 46.068418589953396
      }
     },
     "sealed|choice": {
      "cc": 83.09170328497459,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 90.68804966373514,
       "hard": 76.07466063348416,
       "judge": 86.03351955307262,
       "standard": 88.91489099642814
      }
     },
     "sealed|noul": {
      "cc": 76.04956278293162,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 61.165048543689316,
       "judge": 81.57894736842107,
       "standard": 91.54929577464787
      }
     },
     "sealed|score": {
      "cc": 60.62390898045936,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 72.91119795662057,
       "hard": 56.16498434122372,
       "judge": 63.285909697701555,
       "standard": 59.4051126949867
      }
     }
    }
   },
   "jevbench_score": 71.68461163062156,
   "key": "decisio-gemma-4-31b-v080",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 software; Gemma model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "aminry/decisio@5b42101a191d222062a795194b0aedb5003bf6ea + google/gemma-4-31B-it@842da379",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9596541786743515,
    "decisive_rate": 0.9353099730458221,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.71627256952715,
     "cost": 51.66530928015878,
     "intelligence": 70.42321091828492,
     "speed": 91.18580810090454
    },
    "board_release": "v1.6.43",
    "capability": 79.56974174390604,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 7,
    "capability_rank": 14,
    "composite_ci95": {
     "A": [
      69.85754247280484,
      72.80874617920621
     ],
     "B": [
      68.87932855683242,
      73.03493454427631
     ],
     "C": [
      69.85754247280484,
      72.80874617920621
     ]
    },
    "jevbench_score": 71.6926576652985,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 5,
    "rank": 16,
    "ranks": {
     "A": 16,
     "B": 13,
     "C": 16,
     "capability": 26
    },
    "scores": {
     "A": 71.6926576652985,
     "B": 71.43512018650192,
     "C": 71.6926576652985
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24419243037700653,
     "p50_s_raw": 0.047096215188503265,
     "p95_s_adjusted": 0.3116642528679222,
     "p95_s_raw": 0.0808321264339611
    },
    "views": {
     "Balanced 33:33:33": 67.38267528356181,
     "Emphasis on Accuracy 60:20:20": 68.56682969127965,
     "Emphasis on Cost 20:20:60": 60.07267047173585,
     "Emphasis on Speed 20:60:20": 75.23880708945377,
     "Intelligence only": 70.42321091828492
    }
   },
   "provenance": {
    "carried_metadata_from": "r54/r55/r61 registry and pinned author package",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 18,
   "ranked": true,
   "ranks": {
    "A": 18,
    "B": 15,
    "C": 18,
    "capability": 32
   },
   "repo": "https://github.com/aminry/decisio",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.68461163062156,
    "B": 71.42872937549377,
    "C": 71.68461163062156
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24408713364973664,
    "p50_s_raw": 0.04704356682486832,
    "p95_s_adjusted": 0.315557443536818,
    "p95_s_raw": 0.082778721768409
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.0517072217947,
      90.77969936508954
     ],
     "I": [
      65.27867139621,
      74.3007710998681
     ],
     "capability": [
      75.67457118842692,
      81.98658905786246
     ],
     "composite": {
      "A": [
       69.84990305266089,
       72.80044769282709
      ],
      "B": [
       68.87338684458163,
       73.02825429245821
      ],
      "C": [
       69.84990305266089,
       72.80044769282709
      ]
     },
     "seed": 41
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.71627256952715,
     "capability": 79.56974174390604,
     "intelligence": 70.42321091828492
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 87.69849202278736,
      "cc": 81.41394238511599,
      "cc_by_type": {
       "choice": 81.41394238511599
      },
      "cc_tiered": 76.95832280421416,
      "cc_tiered_open": 70.82494232345373,
      "cc_tiered_sealed": 83.09170328497459,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.41394238511599
     },
     "noul": {
      "accuracy": 0.888,
      "calibration": 90.61159783536232,
      "cc": 77.60000000000001,
      "cc_by_type": {
       "noul": 77.60000000000001
      },
      "cc_tiered": 74.58516786489577,
      "cc_tiered_open": 73.1207729468599,
      "cc_tiered_sealed": 76.04956278293162,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 77.60000000000001
     },
     "score": {
      "accuracy": 0.7093333333333334,
      "calibration": 87.83872785043177,
      "cc": 60.762544130610976,
      "cc_by_type": {
       "score": 60.762544130610976
      },
      "cc_tiered": 59.72614208574488,
      "cc_tiered_open": 58.8283751910304,
      "cc_tiered_sealed": 60.62390898045936,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.762544130610976
     }
    },
    "raw_on_A": {
     "calibration": 84.78496723140272,
     "capability": 77.99664929333233,
     "intelligence": 71.20833135526192
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.37319862058246,
    "Emphasis on Accuracy 60:20:20": 68.56094176838224,
    "Emphasis on Cost 20:20:60": 60.068150950075236,
    "Emphasis on Speed 20:60:20": 75.21754266820024,
    "Intelligence only": 70.42321091828492
   }
  },
  {
   "adapter_id": "harness/specs/jev-1.13.0.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "TypeSafe AI",
   "axes": {
    "calibration": 90.63220203152369,
    "cost": 54.733842199536014,
    "intelligence": 63.62075784024779,
    "speed": 91.21186387733223
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03783700194123639,
      "ece_score": 92.43259961175272,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.43259961175272,
      "score_before_time_limit": 92.43259961175272,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.21293800539083563,
      "brier": 0.06638921832884097,
      "ece": 0.06361185983827498,
      "n": 371,
      "score": 87.277628032345,
      "score_before_time_limit": 87.277628032345,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.24954832881662145,
      "n": 369,
      "nrps": 0.05074999798842498,
      "score": 92.18637845047334,
      "score_before_time_limit": 92.18637845047334,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05276121650105413
     }
    },
    "score": 90.63220203152369
   },
   "capability": 77.12647993588574,
   "class": "jev",
   "composite_ci95": {
    "A": [
     69.09826862399638,
     72.35050977564978
    ],
    "B": [
     66.56025704419686,
     71.32565861066611
    ],
    "C": [
     64.31427798334369,
     72.35050977564978
    ]
   },
   "cost": {
    "basis": "TypeSafe list tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.031780532
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03227542180094787
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jev 1.13.0 (TypeSafe AI)",
   "endpoint_condition": "the operator's hosted API",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 65.60411433522728,
    "I_sealed": 61.637401345268295,
    "base": 63.62075784024779,
    "excess": 1.3933253475207414,
    "gap": 3.9667129899589852,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.0195653974535,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 71.68921044353571,
       "judge": 84.17930125247197,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 54.46376811594203,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 47.82608695652173,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 63.32900949228633,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.45454545454547,
       "hard": 65.44837261503929,
       "judge": 68.91525423728812,
       "standard": 44.148148148148145
      }
     },
     "sealed|choice": {
      "cc": 75.78213638772388,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 76.05498484960461,
       "hard": 73.02036199095022,
       "judge": 75.79143389199255,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 46.14871857614774,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80.0,
       "hard": 28.155339805825253,
       "judge": 47.36842105263157,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 62.981349071933266,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 81.82978723404256,
       "hard": 55.9193404634581,
       "judge": 71.57122119397567,
       "standard": 54.796339024765395
      }
     }
    }
   },
   "jevbench_score": 71.44711653093671,
   "key": "jev-1.13.0",
   "last_measured_on": "2026-10-05",
   "licence": "proprietary API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "jev-1.13.0 (response model id on all 590 successful rows)",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.976027397260274,
    "decisive_rate": 0.7870619946091644,
    "supported": true,
    "valid": 371
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 12,
    "axes": {
     "calibration": 90.63220203152369,
     "cost": 54.733842199536014,
     "intelligence": 63.62075784024779,
     "speed": 91.49989299046987
    },
    "board_release": "v1.6.43",
    "capability": 77.12647993588574,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": 21,
    "composite_ci95": {
     "A": [
      69.1394876572671,
      72.39570149403289
     ],
     "B": [
      66.59085018153897,
      71.36079037075339
     ],
     "C": [
      64.35267606102977,
      72.39570149403289
     ]
    },
    "jevbench_score": 71.49118639284514,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 18,
    "ranks": {
     "A": 18,
     "B": 22,
     "C": 18,
     "capability": 38
    },
    "scores": {
     "A": 71.49118639284514,
     "B": 69.76507971323585,
     "C": 71.49118639284514
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.23910215869545937,
     "p50_s_raw": 0.23910215869545937,
     "p95_s_adjusted": 0.29609236158430574,
     "p95_s_raw": 0.29609236158430574
    },
    "views": {
     "Balanced 33:33:33": 66.78934094542959,
     "Emphasis on Accuracy 60:20:20": 65.48477335742311,
     "Emphasis on Cost 20:20:60": 61.38146766808138,
     "Emphasis on Speed 20:60:20": 74.8779907919869,
     "Intelligence only": 63.620757840247784
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 20,
   "ranked": true,
   "ranks": {
    "A": 20,
    "B": 24,
    "C": 20,
    "capability": 45
   },
   "repo": "https://docs.typesafe.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.44711653093671,
    "B": 69.73150119125978,
    "C": 71.44711653093671
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24289856851100922,
    "p50_s_raw": 0.24289856851100922,
    "p95_s_adjusted": 0.31145023070275785,
    "p95_s_raw": 0.31145023070275785
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "closed",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.83738964744846,
      90.80253311928502
     ],
     "I": [
      57.864503927061655,
      67.65512541027901
     ],
     "capability": [
      72.76645552638956,
      78.36978036921771
     ],
     "composite": {
      "A": [
       69.09826862399638,
       72.35050977564978
      ],
      "B": [
       66.56025704419686,
       71.32565861066611
      ],
      "C": [
       64.31427798334369,
       72.35050977564978
      ]
     },
     "seed": 44
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.63220203152369,
     "capability": 77.12647993588574,
     "intelligence": 63.62075784024779
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8173333333333334,
      "calibration": 92.43259961175272,
      "cc": 76.63954226386139,
      "cc_by_type": {
       "choice": 76.63954226386139
      },
      "cc_tiered": 77.40085089258869,
      "cc_tiered_open": 79.0195653974535,
      "cc_tiered_sealed": 75.78213638772388,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.63954226386139
     },
     "noul": {
      "accuracy": 0.76,
      "calibration": 87.277628032345,
      "cc": 52.0,
      "cc_by_type": {
       "noul": 52.0
      },
      "cc_tiered": 50.30624334604489,
      "cc_tiered_open": 54.46376811594203,
      "cc_tiered_sealed": 46.14871857614774,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 52.0
     },
     "score": {
      "accuracy": 0.752,
      "calibration": 92.18637845047334,
      "cc": 63.113849016480586,
      "cc_by_type": {
       "score": 63.113849016480586
      },
      "cc_tiered": 63.1551792821098,
      "cc_tiered_open": 63.32900949228633,
      "cc_tiered_sealed": 62.981349071933266,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 63.113849016480586
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.73806378390636,
    "Emphasis on Accuracy 60:20:20": 65.45518786046593,
    "Emphasis on Cost 20:20:60": 61.35547295412263,
    "Emphasis on Speed 20:60:20": 74.7620726966441,
    "Intelligence only": 63.620757840247784
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 89.97624968733336,
    "cost": 50.546097335562465,
    "intelligence": 73.40135940607429,
    "speed": 86.35482296351647
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05296499999999975,
      "ece_score": 89.40700000000005,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 89.40700000000005,
      "score_before_time_limit": 89.40700000000005,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.08799999999999997,
      "brier": 0.07739600085333337,
      "ece": 0.046341333333333186,
      "n": 375,
      "score": 90.73173333333337,
      "score_before_time_limit": 90.73173333333337,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.21599999999999997,
      "n": 375,
      "nrps": 0.054102618759999964,
      "score": 89.79001572866669,
      "score_before_time_limit": 89.79001572866669,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07504853333333314
     }
    },
    "score": 89.97624968733336
   },
   "capability": 81.68880454670384,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     69.75293700233391,
     72.11605724026548
    ],
    "B": [
     69.66588646401058,
     72.97501513538305
    ],
    "C": [
     69.75293700233391,
     72.11605724026548
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it list price $0.09/M input, $0.34/M output (the exact base model google/gemma-4-31B-it; listed since Apr 2026), read 2026-10-04 x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04451059729064039
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Large (Chinh Nguyen, Gemma-4-31B decoder, option-letter logits)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 75.23869318243646,
    "I_sealed": 71.56402562971212,
    "base": 73.40135940607429,
    "excess": 1.1012799102861002,
    "gap": 3.674667552724344,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 85.01378955328916,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 84.90091223655237,
       "judge": 80.22412656558997,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 71.3768115942029,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 69.32547839981737,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.654545454545456,
       "hard": 72.09425925925925,
       "judge": 68.68847457627118,
       "standard": 70.5788888888889
      }
     },
     "sealed|choice": {
      "cc": 82.59643939217574,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 80.65610859728505,
       "judge": 81.3780260707635,
       "standard": 88.91489099642814
      }
     },
     "sealed|noul": {
      "cc": 66.16688233643043,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 45.63106796116505,
       "judge": 76.3157894736842,
       "standard": 83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 65.9287551605302,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 73.00248226950355,
       "hard": 67.1347549019608,
       "judge": 65.63335329341317,
       "standard": 60.42299492385786
      }
     }
    }
   },
   "jevbench_score": 71.29893456831563,
   "key": "quyet-1-0-large",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9298245614035088,
    "decisive_rate": 0.912,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.97624968733336,
     "cost": 50.546097335562465,
     "intelligence": 73.40135940607429,
     "speed": 86.87440406834797
    },
    "board_release": "v1.6.43",
    "capability": 81.68880454670384,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 4,
    "capability_rank": 8,
    "composite_ci95": {
     "A": [
      69.83728305350246,
      72.20621881721509
     ],
     "B": [
      69.73317863481549,
      73.04885527628076
     ],
     "C": [
      69.83728305350246,
      72.20621881721509
     ]
    },
    "jevbench_score": 71.38706329141472,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 7,
    "rank": 19,
    "ranks": {
     "A": 19,
     "B": 12,
     "C": 19,
     "capability": 18
    },
    "scores": {
     "A": 71.38706329141472,
     "B": 71.78102942459306,
     "C": 71.38706329141472
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3815564611926675,
     "p50_s_raw": 0.11577823059633374,
     "p95_s_adjusted": 0.5382707504555583,
     "p95_s_raw": 0.19413537522777913
    },
    "views": {
     "Balanced 33:33:33": 66.78760078772265,
     "Emphasis on Accuracy 60:20:20": 69.28473442746802,
     "Emphasis on Cost 20:20:60": 59.18115107083417,
     "Emphasis on Speed 20:60:20": 73.59406855476429,
     "Intelligence only": 73.40135940607429
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 21,
   "ranked": true,
   "ranks": {
    "A": 21,
    "B": 14,
    "C": 21,
    "capability": 21
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Large",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.29893456831563,
    "B": 71.70972890887518,
    "C": 71.29893456831563
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3948847154155374,
    "p50_s_raw": 0.12244235770776868,
    "p95_s_adjusted": 0.5862021130509675,
    "p95_s_raw": 0.21810105652548373
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.58390587934255,
      91.370785899262
     ],
     "I": [
      69.18945219901103,
      76.93636317458929
     ],
     "capability": [
      77.9843219931321,
      83.55754626687737
     ],
     "composite": {
      "A": [
       69.75293700233391,
       72.11605724026548
      ],
      "B": [
       69.66588646401058,
       72.97501513538305
      ],
      "C": [
       69.75293700233391,
       72.11605724026548
      ]
     },
     "seed": 86
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.97624968733336,
     "capability": 81.68880454670384,
     "intelligence": 73.40135940607429
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8706666666666667,
      "calibration": 89.40700000000005,
      "cc": 83.46011386565368,
      "cc_by_type": {
       "choice": 83.46011386565368
      },
      "cc_tiered": 83.80511447273244,
      "cc_tiered_open": 85.01378955328916,
      "cc_tiered_sealed": 82.59643939217574,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.46011386565368
     },
     "noul": {
      "accuracy": 0.848,
      "calibration": 90.73173333333337,
      "cc": 69.6,
      "cc_by_type": {
       "noul": 69.6
      },
      "cc_tiered": 68.77184696531667,
      "cc_tiered_open": 71.3768115942029,
      "cc_tiered_sealed": 66.16688233643043,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.6
     },
     "score": {
      "accuracy": 0.7493333333333333,
      "calibration": 89.79001572866669,
      "cc": 66.83812280701756,
      "cc_by_type": {
       "score": 66.83812280701756
      },
      "cc_tiered": 67.6271167801738,
      "cc_tiered_open": 69.32547839981737,
      "cc_tiered_sealed": 65.9287551605302,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 66.83812280701756
     }
    },
    "raw_on_A": {
     "calibration": 86.55147389375357,
     "capability": 81.17122174668583,
     "intelligence": 75.7909695996181
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.68478111190242,
    "Emphasis on Accuracy 60:20:20": 69.21830455447501,
    "Emphasis on Cost 20:20:60": 59.13267629033431,
    "Emphasis on Speed 20:60:20": 73.36968777818804,
    "Intelligence only": 73.40135940607429
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mark Marosi (Mapika)",
   "axes": {
    "calibration": 80.75417276143466,
    "cost": 57.755449421401664,
    "intelligence": 61.94396690132827,
    "speed": 90.45532474538248
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.048897499999999955,
      "ece_score": 90.22050000000002,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 88.65667800000001,
      "score_before_time_limit": 90.22050000000002,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.013333333333333308,
      "brier": 0.16693694264000003,
      "ece": 0.1691528,
      "n": 375,
      "score": 65.46363264,
      "score_before_time_limit": 66.16944000000001,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.952,
      "interval90_width": 0.33711111111111114,
      "n": 375,
      "nrps": 0.07347944788000003,
      "score": 88.14220764430401,
      "score_before_time_limit": 89.57541427266668,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.06750613333333315
     }
    },
    "score": 80.75417276143466
   },
   "capability": 71.34906983138146,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     67.85725317210733,
     71.69539407751004
    ],
    "B": [
     65.33235658388288,
     70.43453513882069
    ],
    "C": [
     60.755223427667865,
     71.69539407751004
    ]
   },
   "cost": {
    "basis": "add-requests run 46 (v1.5 basis, unpublished) documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025594796798029562
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-12b v2 (Mapika)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000",
   "intelligence": {
    "I_open": 62.171940568331934,
    "I_sealed": 61.7159932343246,
    "base": 61.94396690132827,
    "excess": -2.11744030843091,
    "gap": 0.4559473340073339,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.72393862958435,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 76.26895187870797,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 58.42028985507247,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 53.37159322033899,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 55.93666666666668,
       "hard": 56.19055555555556,
       "judge": 47.0086440677966,
       "standard": 55.995555555555555
      }
     },
     "sealed|choice": {
      "cc": 72.96622719698513,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 66.91176470588235,
       "judge": 77.6536312849162,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 59.149881609534575,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 41.747572815533985,
       "judge": 60.52631578947369,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 53.03187089645412,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.177304964539,
       "hard": 46.133137254901946,
       "judge": 55.79760479041915,
       "standard": 54.60802030456852
      }
     }
    }
   },
   "jevbench_score": 70.30315863416196,
   "key": "decider-12b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Mapika/decider-12b@8ac1efa708b71b86ae33b01d2a8d7a3ddcb48e66",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8333333333333334,
    "decisive_rate": 0.9865229110512129,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.98845142422223,
     "cost": 57.755449421401664,
     "intelligence": 63.01250112161242,
     "speed": 90.47583056481872
    },
    "board_release": "v1.6.43",
    "capability": 72.50047627291733,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 17,
    "capability_rank": 34,
    "composite_ci95": {
     "A": [
      68.54138708333946,
      72.20753634823356
     ],
     "B": [
      66.06197783195658,
      71.11572775091496
     ],
     "C": [
      63.43447024505396,
      72.20753634823356
     ]
    },
    "jevbench_score": 70.87955164338175,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 8,
    "rank": 21,
    "ranks": {
     "A": 21,
     "B": 24,
     "C": 20,
     "capability": 60
    },
    "scores": {
     "A": 70.87955164338175,
     "B": 69.15281897628992,
     "C": 70.87955164338175
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25826345253735783,
     "p50_s_raw": 0.054131726268678904,
     "p95_s_adjusted": 0.3470195889472961,
     "p95_s_raw": 0.09850979447364806
    },
    "views": {
     "Balanced 33:33:33": 67.81664735288307,
     "Emphasis on Accuracy 60:20:20": 65.80968465261019,
     "Emphasis on Cost 20:20:60": 63.39892316578739,
     "Emphasis on Speed 20:60:20": 75.36672286262451,
     "Intelligence only": 63.01250112161242
    }
   },
   "provenance": {
    "carried_metadata_from": "run 46 author metadata + current P83 staged pin/measurement receipt",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 26,
   "ranked": true,
   "ranks": {
    "A": 26,
    "B": 29,
    "C": 24,
    "capability": 77
   },
   "repo": "https://huggingface.co/Mapika/decider-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.30315863416196,
    "B": 68.45557506540338,
    "C": 70.30315863416196
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25852626711130144,
    "p50_s_raw": 0.05426313355565071,
    "p95_s_adjusted": 0.34830751921981573,
    "p95_s_raw": 0.09915375960990787
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.58237996826853,
      82.2031372305055
     ],
     "I": [
      56.7518720519787,
      66.169056099611
     ],
     "capability": [
      66.92392625670921,
      73.80985552870881
     ],
     "composite": {
      "A": [
       67.85725317210733,
       71.69539407751004
      ],
      "B": [
       65.33235658388288,
       70.43453513882069
      ],
      "C": [
       60.755223427667865,
       71.69539407751004
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.75417276143466,
     "capability": 71.34906983138146,
     "intelligence": 61.94396690132827
    },
    "per_type": {
     "choice": {
      "accuracy": 0.792,
      "calibration": 88.65667800000001,
      "cc": 73.39977075301005,
      "cc_by_type": {
       "choice": 73.39977075301005
      },
      "cc_tiered": 73.84508291328474,
      "cc_tiered_open": 74.72393862958435,
      "cc_tiered_sealed": 72.96622719698513,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.39977075301005
     },
     "noul": {
      "accuracy": 0.8133333333333334,
      "calibration": 65.46363264,
      "cc": 62.66666666666667,
      "cc_by_type": {
       "noul": 62.66666666666667
      },
      "cc_tiered": 58.78508573230352,
      "cc_tiered_open": 58.42028985507247,
      "cc_tiered_sealed": 59.149881609534575,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.66666666666667
     },
     "score": {
      "accuracy": 0.6453333333333333,
      "calibration": 88.14220764430401,
      "cc": 53.379105263157925,
      "cc_by_type": {
       "score": 53.379105263157925
      },
      "cc_tiered": 53.20173205839656,
      "cc_tiered_open": 53.37159322033899,
      "cc_tiered_sealed": 53.03187089645412,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.379105263157925
     }
    },
    "raw_on_A": {
     "calibration": 80.60180365288517,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.39575871352413,
    "Emphasis on Accuracy 60:20:20": 65.10380107878295,
    "Emphasis on Cost 20:20:60": 63.177616401087306,
    "Emphasis on Speed 20:60:20": 75.04853863373522,
    "Intelligence only": 61.94396690132827
   }
  },
  {
   "adapter_id": "run/wity-1-auto.spec.json:wity_reasoning_adapter.WityReasoningAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 90.45687167393245,
     "cost": 41.78292309262718,
     "intelligence": 78.56795761907267,
     "speed": 75.05991709119236
    },
    "label": "Base-model reference price (base undisclosed at the author's request)",
    "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens",
    "usd_per_1000": 0.08721059113300492
   },
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Wity (alphanimble)",
   "axes": {
    "calibration": 85.93671111258612,
    "cost": 58.8349720563948,
    "intelligence": 67.88616994428031,
    "speed": 68.32012851304185
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09748974871312502,
      "ece_score": 80.502050257375,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 78.35532891717834,
      "score_before_time_limit": 80.502050257375,
      "time_limit_failed_share": 0.02666666666666667
     },
     "noul": {
      "abstention_rate": 0.09333333333333338,
      "brier": 0.06247285161168309,
      "ece": 0.035695432538436114,
      "n": 375,
      "score": 91.62276797908194,
      "score_before_time_limit": 92.86091349231278,
      "time_limit_failed_share": 0.013333333333333334
     },
     "score": {
      "interval90_coverage": 0.9466666666666667,
      "interval90_width": 0.19822222222222222,
      "n": 375,
      "nrps": 0.052696939614440746,
      "score": 87.83203644149812,
      "score_before_time_limit": 90.48630127901592,
      "time_limit_failed_share": 0.029333333333333333,
      "top_ece": 0.06878851740262039
     }
    },
    "score": 85.93671111258612
   },
   "capability": 76.91144052843322,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     66.90698553569433,
     70.19728667076625
    ],
    "B": [
     65.96748802157725,
     70.62571507703021
    ],
    "C": [
     66.90698553569433,
     70.19728667076625
    ]
   },
   "cost": {
    "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.074026736
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.023559582938388626
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 36,
    "n_over_limit_answered_ok": 36,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "wity-1 (Wity, reasoning auto)",
   "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=auto as the operator asked for their main row.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.6410150463723,
    "I_sealed": 69.13132484218832,
    "base": 67.88616994428031,
    "excess": -5.063697438254261,
    "gap": -2.4903097958160174,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 81.27842432028991,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 84.17930125247197,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 54.76328502415459,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 39.13043478260869,
       "judge": 60.00000000000001,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 63.8813357946724,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 81.26135069496807,
       "hard": 66.56206622564854,
       "judge": 62.42171848967287,
       "standard": 52.01929344007156
      }
     },
     "sealed|choice": {
      "cc": 75.45166028902736,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 63.348416289592755,
       "judge": 81.3780260707635,
       "standard": 87.80638009607095
      }
     },
     "sealed|noul": {
      "cc": 69.00817290045845,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 51.45631067961165,
       "judge": 71.05263157894737,
       "standard": 91.54929577464787
      }
     },
     "sealed|score": {
      "cc": 62.93414133707918,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 88.63984239469578,
       "hard": 49.15477768475961,
       "judge": 70.29595017077045,
       "standard": 66.59730486237312
      }
     }
    }
   },
   "jevbench_score": 68.9647088778163,
   "key": "wity-1",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9408284023668639,
    "decisive_rate": 0.9135135135135135,
    "supported": true,
    "valid": 370
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 14,
    "axes": {
     "calibration": 87.9497550095679,
     "cost": 58.8349720563948,
     "intelligence": 70.28143483604077,
     "speed": 71.79000307789984
    },
    "board_release": "v1.6.43",
    "capability": 79.11559492280433,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      68.86577835236345,
      71.94201428005243
     ],
     "B": [
      67.99651553112257,
      72.43020923475851
     ],
     "C": [
      68.86577835236345,
      71.94201428005243
     ]
    },
    "jevbench_score": 70.76534010331063,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 22,
    "ranks": {
     "A": 22,
     "B": 17,
     "C": 21,
     "capability": 28
    },
    "scores": {
     "A": 70.76534010331063,
     "B": 70.66802669351945,
     "C": 70.76534010331063
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.5746641159057617,
     "p50_s_raw": 1.5746641159057617,
     "p95_s_adjusted": 4.205443101748823,
     "p95_s_raw": 4.205443101748823
    },
    "views": {
     "Balanced 33:33:33": 66.4382396447495,
     "Emphasis on Accuracy 60:20:20": 67.92395209017961,
     "Emphasis on Cost 20:20:60": 63.172705766286356,
     "Emphasis on Speed 20:60:20": 68.48025157142519,
     "Intelligence only": 70.28143483604077
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 35,
   "ranked": true,
   "ranks": {
    "A": 35,
    "B": 28,
    "C": 29,
    "capability": 47
   },
   "repo": "https://wity.alphanimble.com/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.9647088778163,
    "B": 68.74626813567255,
    "C": 68.9647088778163
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.8473002910614014,
    "p50_s_raw": 1.8473002910614014,
    "p95_s_adjusted": 7.969840868189922,
    "p95_s_raw": 7.969840868189922
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Base model: undisclosed (author request)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.89162458329712,
      88.19374313243327
     ],
     "I": [
      62.25329303534609,
      72.47551711346684
     ],
     "capability": [
      72.39990539102567,
      79.72804176993532
     ],
     "composite": {
      "A": [
       66.90698553569433,
       70.19728667076625
      ],
      "B": [
       65.96748802157725,
       70.62571507703021
      ],
      "C": [
       66.90698553569433,
       70.19728667076625
      ]
     },
     "seed": 111
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.93671111258612,
     "capability": 76.91144052843322,
     "intelligence": 67.88616994428031
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8146666666666667,
      "calibration": 78.35532891717834,
      "cc": 76.29851368377176,
      "cc_by_type": {
       "choice": 76.29851368377176
      },
      "cc_tiered": 78.36504230465863,
      "cc_tiered_open": 81.27842432028991,
      "cc_tiered_sealed": 75.45166028902736,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.29851368377176
     },
     "noul": {
      "accuracy": 0.848,
      "calibration": 91.62276797908194,
      "cc": 69.6,
      "cc_by_type": {
       "noul": 69.6
      },
      "cc_tiered": 61.885728962306516,
      "cc_tiered_open": 54.76328502415459,
      "cc_tiered_sealed": 69.00817290045845,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.6
     },
     "score": {
      "accuracy": 0.7866666666666666,
      "calibration": 87.83203644149812,
      "cc": 63.48262267413758,
      "cc_by_type": {
       "score": 63.48262267413758
      },
      "cc_tiered": 63.40773856587579,
      "cc_tiered_open": 63.8813357946724,
      "cc_tiered_sealed": 62.93414133707918,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 63.48262267413758
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.026666666666666616,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.70508264427026,
    "Emphasis on Accuracy 60:20:20": 65.94105907808395,
    "Emphasis on Cost 20:20:60": 62.22187070905237,
    "Emphasis on Speed 20:60:20": 66.10420012709099,
    "Intelligence only": 67.88616994428031
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mark Marosi (Mapika)",
   "axes": {
    "calibration": 82.51700722651287,
    "cost": 57.755449421401664,
    "intelligence": 59.571985034161884,
    "speed": 90.44187377539939
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0649509375000001,
      "ece_score": 87.00981249999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 85.50164241666666,
      "score_before_time_limit": 87.00981249999998,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.040000000000000036,
      "brier": 0.1306117199200001,
      "ece": 0.12608400000000003,
      "n": 375,
      "score": 73.98551253333332,
      "score_before_time_limit": 74.7832,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9893333333333333,
      "interval90_width": 0.493333333333333,
      "n": 375,
      "nrps": 0.07613894289555556,
      "score": 88.06386672953866,
      "score_before_time_limit": 89.49579952188888,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.06697253333333336
     }
    },
    "score": 82.51700722651287
   },
   "capability": 71.04449613033738,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     67.4708863013825,
     71.02991430335766
    ],
    "B": [
     64.47740177392501,
     69.3484390824282
    ],
    "C": [
     56.12328185610373,
     71.02991430335766
    ]
   },
   "cost": {
    "basis": "add-requests run 46 (v1.5 basis, unpublished) documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025594796798029562
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-12b v1, stock Gemma-4-12B-it (Mapika)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000",
   "intelligence": {
    "I_open": 61.530130814752454,
    "I_sealed": 57.61383925357131,
    "base": 59.571985034161884,
    "excess": 1.3429039187429033,
    "gap": 3.916291561181147,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 69.2197724963132,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 60.44825313117996,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 73.89855072463769,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 70.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 41.4720692233065,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.67363636363637,
       "hard": 39.991944444444435,
       "judge": 43.4459322033898,
       "standard": 35.370740740740736
      }
     },
     "sealed|choice": {
      "cc": 66.2478597142713,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 61.82126696832577,
       "judge": 66.4804469273743,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 57.758775648268774,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 30.097087378640786,
       "judge": 65.78947368421053,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 48.83488239817386,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.59943262411348,
       "hard": 41.64539215686275,
       "judge": 54.09401197604791,
       "standard": 46.94289340101522
      }
     }
    }
   },
   "jevbench_score": 69.83686869504557,
   "key": "decider-12b-v1",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Mapika/decider-12b@09e9e3387fa42fed740233be2290385d18a6044d",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.865546218487395,
    "decisive_rate": 0.9622641509433962,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.7629373406296,
     "cost": 57.755449421401664,
     "intelligence": 60.62893554403879,
     "speed": 90.47654290454523
    },
    "board_release": "v1.6.43",
    "capability": 72.1959364423342,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 19,
    "capability_rank": 36,
    "composite_ci95": {
     "A": [
      68.08779334144351,
      71.63200792040908
     ],
     "B": [
      65.27927398150561,
      70.07926650409995
     ],
     "C": [
      58.91555277575465,
      71.63200792040908
     ]
    },
    "jevbench_score": 70.42352588714175,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 10,
    "rank": 25,
    "ranks": {
     "A": 25,
     "B": 26,
     "C": 23,
     "capability": 63
    },
    "scores": {
     "A": 70.42352588714175,
     "B": 68.21936151093459,
     "C": 70.42352588714175
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25808657668530943,
     "p50_s_raw": 0.054043288342654705,
     "p95_s_adjusted": 0.3472004604525864,
     "p95_s_raw": 0.0986002302262932
    },
    "views": {
     "Balanced 33:33:33": 66.87360828773896,
     "Emphasis on Accuracy 60:20:20": 64.22748164585495,
     "Emphasis on Cost 20:20:60": 62.90137869162178,
     "Emphasis on Speed 20:60:20": 74.66484030090665,
     "Intelligence only": 60.62893554403879
    }
   },
   "provenance": {
    "carried_metadata_from": "run 46 author metadata + current P83 staged pin/measurement receipt",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 30,
   "ranked": true,
   "ranks": {
    "A": 30,
    "B": 33,
    "C": 30,
    "capability": 82
   },
   "repo": "https://huggingface.co/Mapika/decider-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.83686869504557,
    "B": 67.51032012820612,
    "C": 68.84404838820772
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2587531372904778,
    "p50_s_raw": 0.054376568645238876,
    "p95_s_adjusted": 0.3490816324949264,
    "p95_s_raw": 0.09954081624746322
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.40473152183768,
      83.88253007662135
     ],
     "I": [
      54.75241389332252,
      63.49771826954563
     ],
     "capability": [
      66.46482011493742,
      72.9669680184571
     ],
     "composite": {
      "A": [
       67.4708863013825,
       71.02991430335766
      ],
      "B": [
       64.47740177392501,
       69.3484390824282
      ],
      "C": [
       56.12328185610373,
       71.02991430335766
      ]
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.51700722651287,
     "capability": 71.04449613033738,
     "intelligence": 59.571985034161884
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7466666666666667,
      "calibration": 85.50164241666666,
      "cc": 67.60228489148659,
      "cc_by_type": {
       "choice": 67.60228489148659
      },
      "cc_tiered": 67.73381610529225,
      "cc_tiered_open": 69.2197724963132,
      "cc_tiered_sealed": 66.2478597142713,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.60228489148659
     },
     "noul": {
      "accuracy": 0.824,
      "calibration": 73.98551253333332,
      "cc": 64.8,
      "cc_by_type": {
       "noul": 64.8
      },
      "cc_tiered": 65.82866318645323,
      "cc_tiered_open": 73.89855072463769,
      "cc_tiered_sealed": 57.758775648268774,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.8
     },
     "score": {
      "accuracy": 0.64,
      "calibration": 88.06386672953866,
      "cc": 47.483307017543865,
      "cc_by_type": {
       "score": 47.483307017543865
      },
      "cc_tiered": 45.153475810740176,
      "cc_tiered_open": 41.4720692233065,
      "cc_tiered_sealed": 48.83488239817386,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.483307017543865
     }
    },
    "raw_on_A": {
     "calibration": 79.88170832181567,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.43396656395934,
    "Emphasis on Accuracy 60:20:20": 63.507829053434584,
    "Emphasis on Cost 20:20:60": 62.667329811932674,
    "Emphasis on Speed 20:60:20": 74.32593063602127,
    "Intelligence only": 59.57198503416188
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 82.93469131589123,
    "cost": 56.36034169226667,
    "intelligence": 60.54320091058725,
    "speed": 91.34133469499314
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07394265024295099,
      "ece_score": 85.2114699514098,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.2114699514098,
      "score_before_time_limit": 85.2114699514098,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.035040431266846306,
      "brier": 0.1393319820198383,
      "ece": 0.13485522993675053,
      "n": 371,
      "score": 73.02895401264989,
      "score_before_time_limit": 73.02895401264989,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.33220415537488723,
      "n": 369,
      "nrps": 0.06922052747100878,
      "score": 90.56364998361398,
      "score_before_time_limit": 90.56364998361398,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05975323642835579
     }
    },
    "score": 82.93469131589123
   },
   "capability": 71.73894611323924,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     67.16941178445903,
     71.41294357282025
    ],
    "B": [
     64.31810039127373,
     69.97094141798148
    ],
    "C": [
     56.54608963400217,
     71.41294357282025
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; no public tariff, customer asked for the base-model reference): OpenRouter google/gemma-3-12b-it list price $0.05/M input, $0.15/M output (gemma-4-12b-it is not listed on OpenRouter, read 2026-10-04; same reference as the Cygnet and Jev-Omni Gemma-4-12B rows) x the shim's own measured usage.prompt_tokens/completion_tokens; self-hosted on 1x H100 80GB (Lium), vLLM 0.30.0 BF16; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02848756157635468
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "torchcast-decision-12b (Torchcast AI, Gemma-4-12B fine-tune, option-letter logprob readout)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order b113eac1)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 59.97536524790964,
    "I_sealed": 61.11103657326486,
    "base": 60.54320091058725,
    "excess": -3.7090589677934624,
    "gap": -1.1356713253552186,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 71.98506861847196,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 56.49758454106281,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 50.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 51.44344258419417,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.22709032709656,
       "hard": 57.48378737898936,
       "judge": 45.79568230097426,
       "standard": 44.44256954798248
      }
     },
     "sealed|choice": {
      "cc": 68.40340626918382,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 60.80316742081447,
       "judge": 72.06703910614526,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 62.37236788127848,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 45.63106796116505,
       "judge": 65.78947368421053,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 52.557335569332295,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.05633413996496,
       "hard": 44.66535408470005,
       "judge": 55.981937871697106,
       "standard": 54.95489579973323
      }
     }
    }
   },
   "jevbench_score": 69.84980331350847,
   "key": "torchcast-decision-12b",
   "last_measured_on": "2026-10-04",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8575418994413407,
    "decisive_rate": 0.9649595687331537,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.93469131589123,
     "cost": 56.36034169226667,
     "intelligence": 60.54320091058725,
     "speed": 91.74573681567192
    },
    "board_release": "v1.6.43",
    "capability": 71.73894611323924,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 20,
    "capability_rank": 37,
    "composite_ci95": {
     "A": [
      67.22388654747238,
      71.4745219700647
     ],
     "B": [
      64.35805126511531,
      70.01822593897899
     ],
     "C": [
      56.59214410663297,
      71.4745219700647
     ]
    },
    "jevbench_score": 69.9087143535159,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 11,
    "rank": 27,
    "ranks": {
     "A": 27,
     "B": 32,
     "C": 25,
     "capability": 64
    },
    "scores": {
     "A": 69.9087143535159,
     "B": 67.81076568849952,
     "C": 69.9087143535159
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2243030747398734,
     "p50_s_raw": 0.037151537369936705,
     "p95_s_adjusted": 0.29825730649754406,
     "p95_s_raw": 0.07412865324877203
    },
    "views": {
     "Balanced 33:33:33": 66.43077398924878,
     "Emphasis on Accuracy 60:20:20": 63.94347960496841,
     "Emphasis on Cost 20:20:60": 61.999556455891316,
     "Emphasis on Speed 20:60:20": 74.67237011934479,
     "Intelligence only": 60.54320091058725
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 29,
   "ranked": true,
   "ranks": {
    "A": 29,
    "B": 32,
    "C": 26,
    "capability": 74
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.84980331350847,
    "B": 67.76641469720529,
    "C": 69.84980331350847
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23964857459068298,
    "p50_s_raw": 0.04482428729534149,
    "p95_s_adjusted": 0.30640205964446066,
    "p95_s_raw": 0.07820102982223033
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.04707649811247,
      84.79039923447887
     ],
     "I": [
      55.00644510399965,
      65.05987282834745
     ],
     "capability": [
      67.02511371513799,
      74.5041047187016
     ],
     "composite": {
      "A": [
       67.16941178445903,
       71.41294357282025
      ],
      "B": [
       64.31810039127373,
       69.97094141798148
      ],
      "C": [
       56.54608963400217,
       71.41294357282025
      ]
     },
     "seed": 106
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.93469131589123,
     "capability": 71.73894611323924,
     "intelligence": 60.54320091058725
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7626666666666667,
      "calibration": 85.2114699514098,
      "cc": 69.64845637202428,
      "cc_by_type": {
       "choice": 69.64845637202428
      },
      "cc_tiered": 70.19423744382789,
      "cc_tiered_open": 71.98506861847196,
      "cc_tiered_sealed": 68.40340626918382,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.64845637202428
     },
     "noul": {
      "accuracy": 0.8186666666666667,
      "calibration": 73.02895401264989,
      "cc": 63.733333333333334,
      "cc_by_type": {
       "noul": 63.733333333333334
      },
      "cc_tiered": 59.43497621117064,
      "cc_tiered_open": 56.49758454106281,
      "cc_tiered_sealed": 62.37236788127848,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 63.733333333333334
     },
     "score": {
      "accuracy": 0.6666666666666666,
      "calibration": 90.56364998361398,
      "cc": 52.778195902361034,
      "cc_by_type": {
       "score": 52.778195902361034
      },
      "cc_tiered": 52.000389076763234,
      "cc_tiered_open": 51.44344258419417,
      "cc_tiered_sealed": 52.557335569332295,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.778195902361034
     }
    },
    "raw_on_A": {
     "calibration": 79.9965213222889,
     "capability": 69.86117451490037,
     "intelligence": 59.725827707511854
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.35986297184556,
    "Emphasis on Accuracy 60:20:20": 63.904041615617075,
    "Emphasis on Cost 20:20:60": 61.96247920356028,
    "Emphasis on Speed 20:60:20": 74.51127093708331,
    "Intelligence only": 60.54320091058725
   }
  },
  {
   "adapter_id": "images/winnow-12b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Eldan Ring",
   "axes": {
    "calibration": 82.95806123333315,
    "cost": 56.556023424877196,
    "intelligence": 59.53366229166759,
    "speed": 86.16642825724779
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11234009752278087,
      "ece_score": 77.53198049544383,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 77.53198049544383,
      "score_before_time_limit": 77.53198049544383,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.14016172506738545,
      "brier": 0.09737227728287662,
      "ece": 0.07944785238178745,
      "n": 371,
      "score": 84.11042952364252,
      "score_before_time_limit": 84.11042952364252,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9214092140921409,
      "interval90_width": 0.2443541102077688,
      "n": 369,
      "nrps": 0.06941649910523198,
      "score": 87.23177368091314,
      "score_before_time_limit": 87.23177368091314,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09297401363825267
     }
    },
    "score": 82.95806123333315
   },
   "capability": 71.24586176250037,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     66.37356850038256,
     70.37369660836514
    ],
    "B": [
     63.500204767152745,
     68.96866110446909
    ],
    "C": [
     53.8217689492573,
     70.37369660836514
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.028062900246305422
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Winnow-12B Q8",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 59.81914414395652,
    "I_sealed": 59.24818043937866,
    "base": 59.53366229166759,
    "excess": -2.0024239378603803,
    "gap": 0.5709637045778635,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.47816196296756,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 68.35860250494396,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 50.49758454106281,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 58.48168592783921,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.310580174513184,
       "hard": 65.34102213505747,
       "judge": 48.44185240673705,
       "standard": 58.40831667171895
      }
     },
     "sealed|choice": {
      "cc": 69.74580750216904,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 62.33031674208144,
       "judge": 76.72253258845438,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 52.512988405651086,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 28.155339805825253,
       "judge": 60.52631578947369,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 55.48574541031586,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 72.39586027967081,
       "hard": 51.82211180030672,
       "judge": 58.61411366152696,
       "standard": 49.665402818839944
      }
     }
    }
   },
   "jevbench_score": 68.80118463059195,
   "key": "winnow-12b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0, including the applicable Gemma 4 base/derivative licence terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9153605015673981,
    "decisive_rate": 0.8598382749326146,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.95806123333315,
     "cost": 56.556023424877196,
     "intelligence": 59.53366229166759,
     "speed": 86.69753831262085
    },
    "board_release": "v1.6.43",
    "capability": 71.24586176250037,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 22,
    "capability_rank": 39,
    "composite_ci95": {
     "A": [
      66.45196246998344,
      70.4618307059656
     ],
     "B": [
      63.55759159512183,
      69.03636277102011
     ],
     "C": [
      53.88529001998626,
      70.4618307059656
     ]
    },
    "jevbench_score": 68.88542163459404,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 13,
    "rank": 30,
    "ranks": {
     "A": 30,
     "B": 36,
     "C": 33,
     "capability": 69
    },
    "scores": {
     "A": 68.88542163459404,
     "B": 66.78718784915615,
     "C": 67.81878724910808
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.38378054890781643,
     "p50_s_raw": 0.1168902744539082,
     "p95_s_adjusted": 0.5573951802216469,
     "p95_s_raw": 0.20369759011082347
    },
    "views": {
     "Balanced 33:33:33": 65.19874575043826,
     "Emphasis on Accuracy 60:20:20": 62.8080780809742,
     "Emphasis on Cost 20:20:60": 61.44293550575936,
     "Emphasis on Speed 20:60:20": 72.37789977811173,
     "Intelligence only": 59.53366229166759
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 36,
   "ranked": true,
   "ranks": {
    "A": 36,
    "B": 38,
    "C": 36,
    "capability": 80
   },
   "repo": "https://huggingface.co/EldanRing/Winnow-12B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.80118463059195,
    "B": 66.72382367986754,
    "C": 67.73585458618234
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3973340891301632,
    "p50_s_raw": 0.1236670445650816,
    "p95_s_adjusted": 0.6084170100279152,
    "p95_s_raw": 0.2292085050139576
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it LoRA fine-tune, merged and exported as Q8_0 GGUF",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.28501959511246,
      85.05351182481425
     ],
     "I": [
      54.0544167472914,
      64.20476080861759
     ],
     "capability": [
      66.87367358483415,
      74.02873882700322
     ],
     "composite": {
      "A": [
       66.37356850038256,
       70.37369660836514
      ],
      "B": [
       63.500204767152745,
       68.96866110446909
      ],
      "C": [
       53.8217689492573,
       70.37369660836514
      ]
     },
     "seed": 109
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.95806123333315,
     "capability": 71.24586176250037,
     "intelligence": 59.53366229166759
    },
    "per_type": {
     "choice": {
      "accuracy": 0.768,
      "calibration": 77.53198049544383,
      "cc": 70.33051353220351,
      "cc_by_type": {
       "choice": 70.33051353220351
      },
      "cc_tiered": 70.1119847325683,
      "cc_tiered_open": 70.47816196296756,
      "cc_tiered_sealed": 69.74580750216904,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.33051353220351
     },
     "noul": {
      "accuracy": 0.7786666666666666,
      "calibration": 84.11042952364252,
      "cc": 55.73333333333332,
      "cc_by_type": {
       "noul": 55.73333333333332
      },
      "cc_tiered": 51.505286473356946,
      "cc_tiered_open": 50.49758454106281,
      "cc_tiered_sealed": 52.512988405651086,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 55.73333333333332
     },
     "score": {
      "accuracy": 0.704,
      "calibration": 87.23177368091314,
      "cc": 56.525484762477454,
      "cc_by_type": {
       "score": 56.525484762477454
      },
      "cc_tiered": 56.98371566907753,
      "cc_tiered_open": 58.48168592783921,
      "cc_tiered_sealed": 55.48574541031586,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.525484762477454
     }
    },
    "raw_on_A": {
     "calibration": 80.53670554364732,
     "capability": 70.27835744009714,
     "intelligence": 60.02000933654696
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.09816233422644,
    "Emphasis on Accuracy 60:20:20": 62.752036163187356,
    "Emphasis on Cost 20:20:60": 61.389302230926255,
    "Emphasis on Speed 20:60:20": 72.15512599343913,
    "Intelligence only": 59.53366229166759
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "krishna765",
   "axes": {
    "calibration": 82.15636027953371,
    "cost": 49.69190284016272,
    "intelligence": 73.02512554551848,
    "speed": 86.37611187702757
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10701823999728471,
      "ece_score": 78.59635200054306,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.59635200054306,
      "score_before_time_limit": 78.59635200054306,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.037735849056603765,
      "brier": 0.059638464379526884,
      "ece": 0.052855966211929055,
      "n": 371,
      "score": 89.42880675761418,
      "score_before_time_limit": 89.42880675761418,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8482384823848238,
      "interval90_width": 0.08897922312556458,
      "n": 369,
      "nrps": 0.06373815398690165,
      "score": 78.44392208044394,
      "score_before_time_limit": 78.44392208044394,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.18369170220210976
     }
    },
    "score": 82.15636027953371
   },
   "capability": 77.5907429125261,
   "composite_ci95": {
    "A": [
     67.02186912560063,
     69.68443115005752
    ],
    "B": [
     67.06183820197873,
     70.80414156770506
    ],
    "C": [
     67.02186912560063,
     69.68443115005752
    ]
   },
   "cost": {
    "basis": "ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it list price $0.09/M input, $0.34/M output (the exact frozen base model google/gemma-4-31B-it; same reference as Quyet-1.0-Large), read 2026-10-04 x the server's own measured usage.input_tokens; letter-mass readout, nothing generated; TorchAO FP8 dynamic on 1x GPU (Lium), deck31b 1.0.0; estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04679784
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04752658090724441
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "deck-31B (krishna765, frozen Gemma-4-31B-it, TorchAO FP8 dynamic)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.23424531362603,
    "I_sealed": 75.81600577741091,
    "base": 73.02512554551848,
    "excess": -8.155148106223118,
    "gap": -5.581760463784875,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.33184897895812,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 64.40342781806196,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 77.39268461007592,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 60.978202351844075,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.27912483548366,
       "hard": 65.74514134257689,
       "judge": 59.223722663075364,
       "standard": 47.92558266171173
      }
     },
     "sealed|choice": {
      "cc": 83.85398241235396,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 90.68804966373514,
       "hard": 76.5837104072398,
       "judge": 87.89571694599628,
       "standard": 88.91489099642814
      }
     },
     "sealed|noul": {
      "cc": 78.05289930692999,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 96.0,
       "hard": 66.99029126213591,
       "judge": 81.57894736842107,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 65.54113561294878,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 75.42107738087628,
       "hard": 61.37893873388316,
       "judge": 71.30554200754482,
       "standard": 60.278948895222165
      }
     }
    }
   },
   "jevbench_score": 68.63044781611312,
   "key": "deck31b",
   "last_measured_on": "2026-10-05",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9439775910364145,
    "decisive_rate": 0.9622641509433962,
    "supported": true,
    "valid": 371
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.15636027953371,
     "cost": 49.69190284016272,
     "intelligence": 73.02512554551848,
     "speed": 86.69970536196755
    },
    "board_release": "v1.6.43",
    "capability": 77.5907429125261,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 10,
    "capability_rank": 19,
    "composite_ci95": {
     "A": [
      67.07103330487536,
      69.73758073723451
     ],
     "B": [
      67.10121070795465,
      70.84803238839223
     ],
     "C": [
      67.07103330487536,
      69.73758073723451
     ]
    },
    "jevbench_score": 68.68200118286292,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 14,
    "rank": 31,
    "ranks": {
     "A": 31,
     "B": 23,
     "C": 27,
     "capability": 35
    },
    "scores": {
     "A": 68.68200118286292,
     "B": 69.34459478009123,
     "C": 68.68200118286292
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4049364184960723,
     "p50_s_raw": 0.12746820924803615,
     "p95_s_adjusted": 0.5280105819925666,
     "p95_s_raw": 0.18900529099628327
    },
    "views": {
     "Balanced 33:33:33": 65.98518077730212,
     "Emphasis on Accuracy 60:20:20": 68.52137025063348,
     "Emphasis on Cost 20:20:60": 58.32800868887079,
     "Emphasis on Speed 20:60:20": 72.85260809439177,
     "Intelligence only": 72.71357044848186
    }
   },
   "provenance": {
    "carried_metadata_from": null,
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 37,
   "ranked": true,
   "ranks": {
    "A": 37,
    "B": 25,
    "C": 31,
    "capability": 42
   },
   "repo": "https://github.com/krishna-gogineni-765/deck31b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.63044781611312,
    "B": 69.30254642091013,
    "C": 68.63044781611312
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4106213027611375,
    "p50_s_raw": 0.13031065138056874,
    "p95_s_adjusted": 0.5609799453057348,
    "p95_s_raw": 0.20548997265286742
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.16081966059086,
      84.92278583205757
     ],
     "I": [
      68.03652261338961,
      76.64977164564989
     ],
     "capability": [
      73.77453269228936,
      80.24074177369353
     ],
     "composite": {
      "A": [
       67.02186912560063,
       69.68443115005752
      ],
      "B": [
       67.06183820197873,
       70.80414156770506
      ],
      "C": [
       67.02186912560063,
       69.68443115005752
      ]
     },
     "seed": 34
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.15636027953371,
     "capability": 77.5907429125261,
     "intelligence": 73.02512554551848
    },
    "per_type": {
     "choice": {
      "accuracy": 0.86,
      "calibration": 78.59635200054306,
      "cc": 82.09599954529521,
      "cc_by_type": {
       "choice": 82.09599954529521
      },
      "cc_tiered": 78.09291569565605,
      "cc_tiered_open": 72.33184897895812,
      "cc_tiered_sealed": 83.85398241235396,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.09599954529521
     },
     "noul": {
      "accuracy": 0.8986666666666666,
      "calibration": 89.42880675761418,
      "cc": 79.73333333333332,
      "cc_by_type": {
       "noul": 79.73333333333332
      },
      "cc_tiered": 77.72279195850295,
      "cc_tiered_open": 77.39268461007592,
      "cc_tiered_sealed": 78.05289930692999,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 79.73333333333332
     },
     "score": {
      "accuracy": 0.72,
      "calibration": 78.44392208044394,
      "cc": 64.7077492239505,
      "cc_by_type": {
       "score": 64.7077492239505
      },
      "cc_tiered": 63.25966898239643,
      "cc_tiered_open": 60.978202351844075,
      "cc_tiered_sealed": 65.54113561294878,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 64.7077492239505
     }
    },
    "raw_on_A": {
     "calibration": 79.57362236322788,
     "capability": 77.19870727297797,
     "intelligence": 74.82379218272806
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.2743590947275,
    "Emphasis on Accuracy 60:20:20": 67.85332390363749,
    "Emphasis on Cost 20:20:60": 57.66447740518504,
    "Emphasis on Speed 20:60:20": 72.04360027906583,
    "Intelligence only": 72.12794492513964
   }
  },
  {
   "adapter_id": "images/cygnet/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "blockbrain",
   "axes": {
    "calibration": 86.96753988280044,
    "cost": 56.42912569481113,
    "intelligence": 54.83928122751086,
    "speed": 91.37843430750362
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08631856803856107,
      "ece_score": 82.73628639228778,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 82.73628639228778,
      "score_before_time_limit": 82.73628639228778,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.1059190859395021,
      "ece": 0.061302788041812155,
      "n": 371,
      "score": 87.73944239163757,
      "score_before_time_limit": 87.73944239163757,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.978319783197832,
      "interval90_width": 0.46589882565492324,
      "n": 369,
      "nrps": 0.07366857377747793,
      "score": 90.42689086447598,
      "score_before_time_limit": 90.42689086447598,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05889680446650118
     }
    },
    "score": 86.96753988280044
   },
   "capability": 70.90341055515565,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     64.37249133234795,
     69.88200973755848
    ],
    "B": [
     60.45647208552507,
     67.40608085289543
    ],
    "C": [
     44.703118980797186,
     67.47914640709492
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.028337561576354687
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Cygnet (blockbrain, frozen Gemma-4-12B-it)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 58.329951877214064,
    "I_sealed": 51.348610577807655,
    "base": 54.83928122751086,
    "excess": 4.407953656968166,
    "gap": 6.9813412994064095,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.62577711735155,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 60.44825313117996,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 57.7536231884058,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 50.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 44.61045532588486,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.78845800105967,
       "hard": 46.33324418399161,
       "judge": 48.54954649888648,
       "standard": 31.66723951258148
      }
     },
     "sealed|choice": {
      "cc": 65.49755292537436,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 56.730769230769226,
       "judge": 66.4804469273743,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 39.63579350399792,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 12.621359223300965,
       "judge": 44.736842105263165,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 48.9124853040507,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.44540243240824,
       "hard": 39.569328601993966,
       "judge": 55.630472156586585,
       "standard": 47.75535986518157
      }
     }
    }
   },
   "jevbench_score": 68.49491799475923,
   "key": "cygnet",
   "last_measured_on": "2026-10-02",
   "licence": "shim MIT; weights Apache-2.0 with Google's Gemma Prohibited Use Policy",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9169435215946844,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.96753988280044,
     "cost": 56.42912569481113,
     "intelligence": 54.83928122751086,
     "speed": 91.77283664982562
    },
    "board_release": "v1.6.43",
    "capability": 70.90341055515565,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 23,
    "capability_rank": 40,
    "composite_ci95": {
     "A": [
      64.42215454949937,
      69.93947552798573
     ],
     "B": [
      60.49166104427309,
      67.44884553134193
     ],
     "C": [
      44.73760732604123,
      67.5344744697005
     ]
    },
    "jevbench_score": 68.55012424151383,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 16,
    "rank": 33,
    "ranks": {
     "A": 33,
     "B": 41,
     "C": 44,
     "capability": 70
    },
    "scores": {
     "A": 68.55012424151383,
     "B": 65.28560070195185,
     "C": 57.26499844882472
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.22363614114001393,
     "p50_s_raw": 0.03681807057000697,
     "p95_s_adjusted": 0.29728592303581536,
     "p95_s_raw": 0.07364296151790768
    },
    "views": {
     "Balanced 33:33:33": 64.03016157894142,
     "Emphasis on Accuracy 60:20:20": 60.00734907512389,
     "Emphasis on Cost 20:20:60": 60.756582833038394,
     "Emphasis on Speed 20:60:20": 72.83760421908612,
     "Intelligence only": 54.83928122751087
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 39,
   "ranked": true,
   "ranks": {
    "A": 39,
    "B": 45,
    "C": 48,
    "capability": 83
   },
   "repo": "https://github.com/blockbrain-ai/cygnet-recipe",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.49491799475923,
    "B": 65.2455343492583,
    "C": 57.218880580042615
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23948653005063533,
    "p50_s_raw": 0.04474326502531767,
    "p95_s_adjusted": 0.3040013258811086,
    "p95_s_raw": 0.07700066294055431
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.02033013748918,
      88.42190039096631
     ],
     "I": [
      49.43932563439576,
      59.02934294021137
     ],
     "capability": [
      66.3438265961016,
      72.89464887942813
     ],
     "composite": {
      "A": [
       64.37249133234795,
       69.88200973755848
      ],
      "B": [
       60.45647208552507,
       67.40608085289543
      ],
      "C": [
       44.703118980797186,
       67.47914640709492
      ]
     },
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.96753988280044,
     "capability": 70.90341055515565,
     "intelligence": 54.83928122751086
    },
    "per_type": {
     "choice": {
      "accuracy": 0.748,
      "calibration": 82.73628639228778,
      "cc": 67.7727991815314,
      "cc_by_type": {
       "choice": 67.7727991815314
      },
      "cc_tiered": 69.06166502136296,
      "cc_tiered_open": 72.62577711735155,
      "cc_tiered_sealed": 65.49755292537436,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.7727991815314
     },
     "noul": {
      "accuracy": 0.736,
      "calibration": 87.73944239163757,
      "cc": 47.199999999999996,
      "cc_by_type": {
       "noul": 47.199999999999996
      },
      "cc_tiered": 48.69470834620186,
      "cc_tiered_open": 57.7536231884058,
      "cc_tiered_sealed": 39.63579350399792,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 47.199999999999996
     },
     "score": {
      "accuracy": 0.6133333333333333,
      "calibration": 90.42689086447598,
      "cc": 48.18037785294212,
      "cc_by_type": {
       "score": 48.18037785294212
      },
      "cc_tiered": 46.76147031496778,
      "cc_tiered_open": 44.61045532588486,
      "cc_tiered_sealed": 48.9124853040507,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.18037785294212
     }
    },
    "raw_on_A": {
     "calibration": 83.24254074674259,
     "capability": 69.54334852201868,
     "intelligence": 55.844156297294745
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.96595288528468,
    "Emphasis on Accuracy 60:20:20": 59.97349776879128,
    "Emphasis on Cost 20:20:60": 60.721881179442136,
    "Emphasis on Speed 20:60:20": 72.68820357974963,
    "Intelligence only": 54.83928122751087
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amin Roudaki (@aminry)",
   "axes": {
    "calibration": 89.15794375860679,
    "cost": 59.34583145470655,
    "intelligence": 52.324988528776075,
    "speed": 86.316096866496
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07110019971983926,
      "ece_score": 85.77996005603214,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.77996005603214,
      "score_before_time_limit": 85.77996005603214,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18328840970350402,
      "brier": 0.10602428963864989,
      "ece": 0.056082845451344523,
      "n": 371,
      "score": 88.7834309097311,
      "score_before_time_limit": 88.7834309097311,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.49186991869918684,
      "n": 369,
      "nrps": 0.073791213570342,
      "score": 92.91044031005717,
      "score_before_time_limit": 92.91044031005717,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03399999011425735
     }
    },
    "score": 89.15794375860679
   },
   "capability": 70.74146614369144,
   "class": "Jev-compatible decision readout on frozen base",
   "composite_ci95": {
    "A": [
     57.29140177846576,
     69.54155861089073
    ],
    "B": [
     53.12522431622021,
     66.49933617015681
    ],
    "C": [
     39.785695679490125,
     61.6003253038042
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist. The reference is the one the rows PUBLISHED on this same base use - cygnet, jev-omni and winnow-12b - OpenRouter google/gemma-3-12b-it, the nearest publicly hosted 12B Gemma sibling, at its FULL catalog pair USD 0.05 per 1M input and USD 0.15 per 1M output. PREREG-R54-AMENDMENT-1 uses the full pair rather than run 53's output=0, so a row that emits tokens pays for them; where a row generates nothing the output term is zero anyway. x this row's own measured input/output tokens, the deck31b method. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.022306333333333338
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.022653689911983754
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisio v0.8.0 on gemma-4-12B-it (frozen, one forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/aminry/decisio tag v0.8.0 (5b42101a191d222062a795194b0aedb5003bf6ea, verified against the GitHub refs API), installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version, and served with their own `python -m decisio.serve.vllm_engine --base gemma-4-12b`. --base carries google/gemma-4-12B-it@707f0a3b (bf16, 22.8 GiB) and that profile's measured settings - temperature 3.592 for every question type, a system turn, the chat template's own answer position, every single-token form of each letter summed. Frozen base: no fine-tuning, no adapter, no task registered. One RTX 6000 Ada 48 GB (Lium, Toronto). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 54, GitHub #169",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 6000 Ada",
   "intelligence": {
    "I_open": 53.076253329484096,
    "I_sealed": 51.57372372806806,
    "base": 52.324988528776075,
    "excess": -1.0708580410222077,
    "gap": 1.502529601416036,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.23658792816236,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 60.44825313117996,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 43.54106280193237,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 39.13043478260869,
       "judge": 30.000000000000004,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 43.45110925835756,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.14556472904645,
       "hard": 43.95984283175309,
       "judge": 46.337851474151414,
       "standard": 33.75630105253128
      }
     },
     "sealed|choice": {
      "cc": 65.8737458551212,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 85.3669351858695,
       "hard": 57.74886877828053,
       "judge": 67.41154562383613,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 40.79862969333631,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 8.737864077669899,
       "judge": 47.36842105263157,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 48.04879563574668,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.31002438791882,
       "hard": 37.90772388734433,
       "judge": 56.21653541756339,
       "standard": 46.44871508374021
      }
     }
    }
   },
   "jevbench_score": 68.06994454095005,
   "key": "decisio-gemma-4-12b-v080",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 software; Gemma model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "aminry/decisio@5b42101a191d222062a795194b0aedb5003bf6ea + google/gemma-4-12B-it@707f0a3b",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9075907590759076,
    "decisive_rate": 0.816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.15794375860679,
     "cost": 59.34583145470655,
     "intelligence": 52.324988528776075,
     "speed": 86.9860428376769
    },
    "board_release": "v1.6.43",
    "capability": 70.74146614369144,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 24,
    "capability_rank": 42,
    "composite_ci95": {
     "A": [
      57.374888808052155,
      69.64960281748938
     ],
     "B": [
      53.18263176533343,
      66.57834556173977
     ],
     "C": [
      39.84367278336956,
      61.696012278542455
     ]
    },
    "jevbench_score": 68.17346094077045,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 17,
    "rank": 36,
    "ranks": {
     "A": 36,
     "B": 45,
     "C": 48,
     "capability": 72
    },
    "scores": {
     "A": 68.17346094077045,
     "B": 64.2795916317687,
     "C": 51.847894540473604
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3548810470150784,
     "p50_s_raw": 0.10244052350753918,
     "p95_s_adjusted": 0.564043904747814,
     "p95_s_raw": 0.20702195237390697
    },
    "views": {
     "Balanced 33:33:33": 63.21404479691977,
     "Emphasis on Accuracy 60:20:20": 58.35636046255001,
     "Emphasis on Cost 20:20:60": 61.60778501333468,
     "Emphasis on Speed 20:60:20": 70.97231847224265,
     "Intelligence only": 52.324988528776075
    }
   },
   "provenance": {
    "carried_metadata_from": "r54/r55/r61 registry and pinned author package",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 42,
   "ranked": true,
   "ranks": {
    "A": 42,
    "B": 51,
    "C": 54,
    "capability": 85
   },
   "repo": "https://github.com/aminry/decisio",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.06994454095005,
    "B": 64.20594095677818,
    "C": 51.76916731572905
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3814489560085349,
    "p50_s_raw": 0.11572447800426744,
    "p95_s_adjusted": 0.6122853703564033,
    "p95_s_raw": 0.23114268517820163
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.58456964369535,
      89.43375634811495
     ],
     "I": [
      46.8581794615679,
      56.47360933655171
     ],
     "capability": [
      66.07776568391779,
      72.37388803479945
     ],
     "composite": {
      "A": [
       57.29140177846576,
       69.54155861089073
      ],
      "B": [
       53.12522431622021,
       66.49933617015681
      ],
      "C": [
       39.785695679490125,
       61.6003253038042
      ]
     },
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.15794375860679,
     "capability": 70.74146614369144,
     "intelligence": 52.324988528776075
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7493333333333333,
      "calibration": 85.77996005603214,
      "cc": 67.9433134715762,
      "cc_by_type": {
       "choice": 67.9433134715762
      },
      "cc_tiered": 69.05516689164179,
      "cc_tiered_open": 72.23658792816236,
      "cc_tiered_sealed": 65.8737458551212,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.9433134715762
     },
     "noul": {
      "accuracy": 0.7333333333333333,
      "calibration": 88.7834309097311,
      "cc": 46.66666666666666,
      "cc_by_type": {
       "noul": 46.66666666666666
      },
      "cc_tiered": 42.16984624763434,
      "cc_tiered_open": 43.54106280193237,
      "cc_tiered_sealed": 40.79862969333631,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.66666666666666
     },
     "score": {
      "accuracy": 0.616,
      "calibration": 92.91044031005717,
      "cc": 47.02582727916151,
      "cc_by_type": {
       "score": 47.02582727916151
      },
      "cc_tiered": 45.749952447052124,
      "cc_tiered_open": 43.45110925835756,
      "cc_tiered_sealed": 48.04879563574668,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.02582727916151
     }
    },
    "raw_on_A": {
     "calibration": 85.4557717128137,
     "capability": 69.50074934822587,
     "intelligence": 53.54572698363802
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.09541643107953,
    "Emphasis on Accuracy 60:20:20": 58.295651506359086,
    "Emphasis on Cost 20:20:60": 61.540126509953886,
    "Emphasis on Speed 20:60:20": 70.70367231906657,
    "Intelligence only": 52.324988528776075
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amin Roudaky / @aminry (roudaky@gmail.com)",
   "axes": {
    "calibration": 87.60965425617285,
    "cost": 59.34583145470655,
    "intelligence": 52.299106200484864,
    "speed": 85.2750295056469
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08060951899685599,
      "ece_score": 83.8780962006288,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 83.8780962006288,
      "score_before_time_limit": 83.8780962006288,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.10747754831036967,
      "ece": 0.06459999777321335,
      "n": 371,
      "score": 87.08000044535733,
      "score_before_time_limit": 87.08000044535733,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.991869918699187,
      "interval90_width": 0.4905149051490515,
      "n": 369,
      "nrps": 0.07376404038589723,
      "score": 91.87086612253242,
      "score_before_time_limit": 91.87086612253242,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.044409318581727114
     }
    },
    "score": 87.60965425617285
   },
   "capability": 69.95438022832886,
   "class": "Jev-compatible decision readout on frozen base",
   "composite_ci95": {
    "A": [
     54.30828550432116,
     69.46579072589009
    ],
    "B": [
     50.243537053294915,
     66.58758109477095
    ],
    "C": [
     37.714087155778586,
     62.67786529075028
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff), CARRIED AND NOT NEW: the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist, and (0.05, 0.15) is the reference the rows PUBLISHED on this same base already use - cygnet, jev-omni, winnow-12b and torchcast-decision-12b - OpenRouter google/gemma-3-12b-it, the nearest publicly hosted 12B Gemma sibling, at its FULL catalog pair USD 0.05 per 1M input and USD 0.15 per 1M output. It is also the rate THIS ROW'S OWN v0.8.0 SIBLING carries, so the two versions are priced identically and the comparison between them is clean. PREREG-R54-AMENDMENT-1's full pair rather than output=0, x this row's own measured input/output tokens, the deck31b method. The server reports its own usage.input_tokens/output_tokens on every answer (664,759 in and 1,477 out over the answered rows: one prefill, one option-label readout step per decision), so no token-accounting amendment was needed. Estimated, not charged.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.022306333333333338
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.022653689911983754
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisio v0.9.0 on gemma-4-12B-it (frozen, one prefill per question, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 21:11:36-21:15:32Z on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,477 of 1,500 answered. github.com/aminry/decisio tag v0.9.0 = 333a0a849de7658969a8f8933b34a8097a8c3489 (verified against the GitHub refs API before the run and re-read as the pod's checked-out HEAD after it), Apache-2.0, installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version (decisio 0.9.0, torch 2.13.0+cu130, vllm 0.30.0, transformers 5.18.0) and served with their own command `python -m decisio.serve.vllm_engine --base gemma-4-12b`. --base carries google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 (bf16, 22.8 GiB) and that profile's own settings - temperature 3.592 for every question type, a system turn, the spaced layout read at the chat template's own answer position, every single-token form of each letter summed. Frozen base: no fine-tuning, no adapter, no task registered. UNCHANGED typesafe adapter and UNCHANGED run_v16.py; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. A FOLLOW-UP OF THIS JOB'S OWN decisio-gemma-4-12b-v080 ROW (run 54, same pool, same basis, same rate), measured as a CANDIDATE: which version is listed is the release lane's decision. WHAT v0.9.0 CHANGES is prefix-cache state-boundary registration, request ticketing and engine-health probing; the whole readout package, serve/systemone.py, serve/temperature.py, serve/abstention.py and the entire vllm_plugin are BYTE-IDENTICAL between the two tags, and no temperature or prompt line moved (r61/receipts/DECISIO-V090-SOURCE-DIFF.json). THE 23 LONG ITEMS FAIL WITH A BARE HTTP 500 from the author's own server - vLLM raises VLLMValidationError because the rendered prompt exceeds the 32,768-token context the profile declares, and the server does not translate that into the 422 the harness scores as a refusal. They are counted wrong either way and they are exactly the 23 items the v1.6.1 common cost basis excludes (the ones the Jev reference itself refuses), but it is the author's own code and A FINDING TO REPORT, NOT A THING TO PATCH AROUND; no setting of theirs was changed and no retry was made. One NVIDIA RTX 6000 Ada Generation 48 GB (Lium, Toronto, CANADA) - the same card model and city on which run 54 measured the v0.8.0 row, chosen over the 96 GB class the author names because the 12B profile is proven to fit it. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline after setup, credential-free. add-requests run 61, PREREG-R61-AMENDMENT-2, benchmarkheaven.com/submit ref 65B71BD3 (bh-submit f05c3298-702f-4314-bc8e-f823d9bce858); previously GitHub #169 for v0.8.0",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 6000 Ada Generation",
   "intelligence": {
    "I_open": 53.066754472907135,
    "I_sealed": 51.53145792806259,
    "base": 52.299106200484864,
    "excess": -1.038091097593702,
    "gap": 1.5352965448445417,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.23658792816236,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 60.44825313117996,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 43.54106280193237,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 39.13043478260869,
       "judge": 30.000000000000004,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 43.422612688626685,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.00297563421439,
       "hard": 44.200521547084826,
       "judge": 45.87070740823223,
       "standard": 33.90447141950822
      }
     },
     "sealed|choice": {
      "cc": 65.83450996318003,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 56.730769230769226,
       "judge": 68.34264432029796,
       "standard": 71.17871659071314
      }
     },
     "sealed|noul": {
      "cc": 40.79862969333631,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 8.737864077669899,
       "judge": 47.36842105263157,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 47.96123412767146,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.87191488571192,
       "hard": 38.251545396524136,
       "judge": 55.512516311610604,
       "standard": 46.59834793503713
      }
     }
    }
   },
   "jevbench_score": 67.66792837235077,
   "key": "decisio-gemma-4-12b-v090",
   "last_measured_on": "2026-10-07",
   "licence": "Apache-2.0 software; Gemma model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "aminry/decisio@333a0a849de7658969a8f8933b34a8097a8c3489 + google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9136212624584718,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.60965425617285,
     "cost": 59.34583145470655,
     "intelligence": 52.299106200484864,
     "speed": 85.86778537116794
    },
    "board_release": "v1.6.43",
    "capability": 69.95438022832886,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 27,
    "capability_rank": 47,
    "composite_ci95": {
     "A": [
      54.37922382108821,
      69.5635857308408
     ],
     "B": [
      50.29209403611676,
      66.65944464115974
     ],
     "C": [
      37.763349875755715,
      62.76631856626907
     ]
    },
    "jevbench_score": 67.76072338665118,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 19,
    "rank": 39,
    "ranks": {
     "A": 39,
     "B": 46,
     "C": 49,
     "capability": 79
    },
    "scores": {
     "A": 67.76072338665118,
     "B": 63.97786249241387,
     "C": 51.483026133201925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4135970369447023,
     "p50_s_raw": 0.13179851847235113,
     "p95_s_adjusted": 0.6261004906438755,
     "p95_s_raw": 0.23805024532193778
    },
    "views": {
     "Balanced 33:33:33": 63.00273578153404,
     "Emphasis on Accuracy 60:20:20": 58.2353176446816,
     "Emphasis on Cost 20:20:60": 61.48719355296207,
     "Emphasis on Speed 20:60:20": 70.51330683201019,
     "Intelligence only": 52.299106200484864
    }
   },
   "provenance": {
    "carried_metadata_from": "r54/r55/r61 registry and pinned author package",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 45,
   "ranked": true,
   "ranks": {
    "A": 45,
    "B": 52,
    "C": 56,
    "capability": 93
   },
   "repo": "https://github.com/aminry/decisio",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.66792837235077,
    "B": 63.9116616447286,
    "C": 51.41252263342376
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.43738937998423355,
    "p50_s_raw": 0.14369468999211676,
    "p95_s_adjusted": 0.6786233761755284,
    "p95_s_raw": 0.2643116880877642
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.75352521796918,
      88.88216270564548
     ],
     "I": [
      45.89453540642751,
      56.94412894091952
     ],
     "capability": [
      65.24251668071038,
      72.31935391386574
     ],
     "composite": {
      "A": [
       54.30828550432116,
       69.46579072589009
      ],
      "B": [
       50.243537053294915,
       66.58758109477095
      ],
      "C": [
       37.714087155778586,
       62.67786529075028
      ]
     },
     "seed": 40
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.60965425617285,
     "capability": 69.95438022832886,
     "intelligence": 52.299106200484864
    },
    "per_type": {
     "choice": {
      "accuracy": 0.748,
      "calibration": 83.8780962006288,
      "cc": 67.7727991815314,
      "cc_by_type": {
       "choice": 67.7727991815314
      },
      "cc_tiered": 69.0355489456712,
      "cc_tiered_open": 72.23658792816236,
      "cc_tiered_sealed": 65.83450996318003,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.7727991815314
     },
     "noul": {
      "accuracy": 0.7333333333333333,
      "calibration": 87.08000044535733,
      "cc": 46.66666666666666,
      "cc_by_type": {
       "noul": 46.66666666666666
      },
      "cc_tiered": 42.16984624763434,
      "cc_tiered_open": 43.54106280193237,
      "cc_tiered_sealed": 40.79862969333631,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.66666666666666
     },
     "score": {
      "accuracy": 0.6133333333333333,
      "calibration": 91.87086612253242,
      "cc": 47.01542304941684,
      "cc_by_type": {
       "score": 47.01542304941684
      },
      "cc_tiered": 45.69192340814907,
      "cc_tiered_open": 43.422612688626685,
      "cc_tiered_sealed": 47.96123412767146,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.01542304941684
     }
    },
    "raw_on_A": {
     "calibration": 83.89503312669699,
     "capability": 68.67777484474954,
     "intelligence": 53.460516562802084
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 62.8958097162103,
    "Emphasis on Accuracy 60:20:20": 58.180462503973104,
    "Emphasis on Cost 20:20:60": 61.42604433104223,
    "Emphasis on Speed 20:60:20": 70.27263111922092,
    "Intelligence only": 52.299106200484864
   }
  },
  {
   "adapter_id": "images/jev-omni/spec.json:omni_v15.JevOmni",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "akhilaaa3",
   "axes": {
    "calibration": 87.04450672835743,
    "cost": 56.05113155939739,
    "intelligence": 55.530225828826175,
    "speed": 84.73829853475769
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10691343524556984,
      "ece_score": 78.61731295088603,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.61731295088603,
      "score_before_time_limit": 78.61731295088603,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.25876010781671155,
      "brier": 0.09426879548573859,
      "ece": 0.04284927173453476,
      "n": 371,
      "score": 91.43014565309304,
      "score_before_time_limit": 91.43014565309304,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.94579945799458,
      "interval90_width": 0.32836495031616986,
      "n": 369,
      "nrps": 0.06732883113404022,
      "score": 91.08606158109325,
      "score_before_time_limit": 91.08606158109325,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0554749686220474
     }
    },
    "score": 87.04450672835743
   },
   "capability": 71.2873662785918,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     65.14989569142833,
     68.85085112761337
    ],
    "B": [
     61.539228888446246,
     66.64844097527748
    ],
    "C": [
     45.9072498011043,
     67.21266709144554
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02917173645320197
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jev-Omni (akhilaaa3, Gemma-4-12B merged)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 59.14991720973798,
    "I_sealed": 51.91053444791438,
    "base": 55.530225828826175,
    "excess": 4.6659951193853555,
    "gap": 7.239382761823599,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.02831016136747,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 73.57659641396666,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 45.118702553485164,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 65.21739130434783,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 57.30273891436131,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.591582585888155,
       "hard": 52.24401347123031,
       "judge": 64.7750828511346,
       "standard": 55.56725205969997
      }
     },
     "sealed|choice": {
      "cc": 68.8767390165768,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 62.33031674208144,
       "judge": 74.86033519553072,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 32.45047104291564,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": 12.621359223300965,
       "judge": 36.8421052631579,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 54.404393284250695,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 74.52093494591816,
       "hard": 48.39164613508641,
       "judge": 58.117448525933035,
       "standard": 50.80203388922203
      }
     }
    }
   },
   "jevbench_score": 67.6378575262987,
   "key": "jev-omni",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0, following Gemma 4; dataset rights stated separately by the author",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9381818181818182,
    "decisive_rate": 0.7412398921832885,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.04450672835743,
     "cost": 56.05113155939739,
     "intelligence": 55.530225828826175,
     "speed": 85.43173201875288
    },
    "board_release": "v1.6.43",
    "capability": 71.2873662785918,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 21,
    "capability_rank": 38,
    "composite_ci95": {
     "A": [
      65.25170208143348,
      68.96455653744843
     ],
     "B": [
      61.61186910919615,
      66.73364708003527
     ],
     "C": [
      45.97947208044005,
      67.32404729169104
     ]
    },
    "jevbench_score": 67.74758858028281,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 20,
    "rank": 40,
    "ranks": {
     "A": 40,
     "B": 42,
     "C": 42,
     "capability": 68
    },
    "scores": {
     "A": 67.74758858028281,
     "B": 64.89216659285307,
     "C": 58.02968592150493
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.45938353668898346,
     "p50_s_raw": 0.15469176834449172,
     "p95_s_adjusted": 0.6232343347743153,
     "p95_s_raw": 0.23661716738715766
    },
    "views": {
     "Balanced 33:33:33": 63.085756293696,
     "Emphasis on Accuracy 60:20:20": 59.82955470935818,
     "Emphasis on Cost 20:20:60": 60.07014592127774,
     "Emphasis on Speed 20:60:20": 70.45744510674369,
     "Intelligence only": 55.53022582882618
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 46,
   "ranked": true,
   "ranks": {
    "A": 46,
    "B": 47,
    "C": 46,
    "capability": 79
   },
   "repo": "https://huggingface.co/akhilaaa3/Jev-Omni",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.6378575262987,
    "B": 64.81159520602941,
    "C": 57.9356949953041
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4731816291809082,
    "p50_s_raw": 0.1615908145904541,
    "p95_s_adjusted": 0.7098102578893304,
    "p95_s_raw": 0.2799051289446652
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it fine-tuned and merged, with a trained 256-way decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.22517864881327,
      88.34500746248735
     ],
     "I": [
      50.30228764136711,
      59.27245651053757
     ],
     "capability": [
      67.20628773376218,
      73.03534010513394
     ],
     "composite": {
      "A": [
       65.14989569142833,
       68.85085112761337
      ],
      "B": [
       61.539228888446246,
       66.64844097527748
      ],
      "C": [
       45.9072498011043,
       67.21266709144554
      ]
     },
     "seed": 46
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.04450672835743,
     "capability": 71.2873662785918,
     "intelligence": 55.530225828826175
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7693333333333333,
      "calibration": 78.61731295088603,
      "cc": 70.5010278222483,
      "cc_by_type": {
       "choice": 70.5010278222483
      },
      "cc_tiered": 71.95252458897214,
      "cc_tiered_open": 75.02831016136747,
      "cc_tiered_sealed": 68.8767390165768,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.5010278222483
     },
     "noul": {
      "accuracy": 0.688,
      "calibration": 91.43014565309304,
      "cc": 37.59999999999999,
      "cc_by_type": {
       "noul": 37.59999999999999
      },
      "cc_tiered": 38.7845867982004,
      "cc_tiered_open": 45.118702553485164,
      "cc_tiered_sealed": 32.45047104291564,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 37.59999999999999
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 91.08606158109325,
      "cc": 55.14278895136254,
      "cc_by_type": {
       "score": 55.14278895136254
      },
      "cc_tiered": 55.85356609930601,
      "cc_tiered_open": 57.30273891436131,
      "cc_tiered_sealed": 54.404393284250695,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.14278895136254
     }
    },
    "raw_on_A": {
     "calibration": 85.06452164862384,
     "capability": 71.04516964846138,
     "intelligence": 57.025817648298926
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 62.95894050468547,
    "Emphasis on Accuracy 60:20:20": 59.76105796949569,
    "Emphasis on Cost 20:20:60": 60.001097502935124,
    "Emphasis on Speed 20:60:20": 70.17328975738995,
    "Intelligence only": 55.53022582882618
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Juspay",
   "axes": {
    "calibration": 84.8928101186779,
    "cost": 50.69588844260597,
    "intelligence": 57.47797916441928,
    "speed": 90.84255575155115
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12367230463806947,
      "ece_score": 75.2655390723861,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 73.96093639513141,
      "score_before_time_limit": 75.2655390723861,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.19733333333333336,
      "brier": 0.08803851800016456,
      "ece": 0.03517045741230692,
      "n": 375,
      "score": 91.9742721600182,
      "score_before_time_limit": 92.96590851753862,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9573333333333334,
      "interval90_width": 0.31933333333333314,
      "n": 375,
      "nrps": 0.07179731457956051,
      "score": 88.7432218008841,
      "score_before_time_limit": 90.18620101715864,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.0622393325386334
     }
    },
    "score": 84.8928101186779
   },
   "capability": 71.18539464154858,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     63.84380091743296,
     68.17521491005895
    ],
    "B": [
     60.77468333880452,
     66.74957560085348
    ],
    "C": [
     46.32022510115811,
     68.17521491005895
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, the M2 reference of the published surogate-rune-26b-a4b-v3 row (the frozen module has no explicit listing-decision line for this base: release-lane line), USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.15323633999999997
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04400179417738659
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Xor 26B-A4B (Juspay, Gemma-4-26B-A4B, bf16)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB (the author's card class), the author's own serving bundle (SGLang image pinned by digest + their compatibility server, single GPU via their documented switch), offline (all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 57.66122558340351,
    "I_sealed": 57.294732745435056,
    "base": 57.47797916441928,
    "excess": -2.2068948044697927,
    "gap": 0.3664928379684511,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.77679085167708,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396666,
       "judge": 76.26895187870797,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 43.24154589371981,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 54.96534000481365,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.747368826312524,
       "hard": 60.18508130733947,
       "judge": 63.88049297699079,
       "standard": 33.26211353074683
      }
     },
     "sealed|choice": {
      "cc": 73.61328423614917,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 67.42081447963801,
       "judge": 77.6536312849162,
       "standard": 73.39573839142753
      }
     },
     "sealed|noul": {
      "cc": 45.734855736359904,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80.0,
       "hard": 22.330097087378633,
       "judge": 50.0,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 52.53605826379611,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.57630209327414,
       "hard": 46.082817980300675,
       "judge": 62.2590525874285,
       "standard": 43.83792543059939
      }
     }
    }
   },
   "jevbench_score": 66.76564670311207,
   "key": "xor-26b-a4b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "juspay/Xor-26B-A4B@8afd4396492084c7c91846ed3791161d54665d34",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.939799331103679,
    "decisive_rate": 0.8059299191374663,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.13921620236111,
     "cost": 50.69588844260597,
     "intelligence": 58.673823517178164,
     "speed": 91.03179229063062
    },
    "board_release": "v1.6.43",
    "capability": 72.40651985976965,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 18,
    "capability_rank": 35,
    "composite_ci95": {
     "A": [
      64.63565801053839,
      68.71970276219201
     ],
     "B": [
      61.75049706211547,
      67.5877402852976
     ],
     "C": [
      49.28381430026326,
      68.71970276219201
     ]
    },
    "jevbench_score": 67.3818936950357,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 22,
    "rank": 42,
    "ranks": {
     "A": 42,
     "B": 40,
     "C": 37,
     "capability": 61
    },
    "scores": {
     "A": 67.3818936950357,
     "B": 65.43945552763522,
     "C": 64.43613635369312
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26713270749896767,
     "p50_s_raw": 0.058566353749483824,
     "p95_s_adjusted": 0.2951846059411764,
     "p95_s_raw": 0.0725923029705882
    },
    "views": {
     "Balanced 33:33:33": 62.82194404983342,
     "Emphasis on Accuracy 60:20:20": 61.0942481872419,
     "Emphasis on Cost 20:20:60": 57.336197993965754,
     "Emphasis on Speed 20:60:20": 71.71094828654465,
     "Intelligence only": 58.67382351717816
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a7 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 49,
   "ranked": true,
   "ranks": {
    "A": 49,
    "B": 49,
    "C": 45,
    "capability": 81
   },
   "repo": "https://huggingface.co/juspay/Xor-26B-A4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.76564670311207,
    "B": 64.6755084312461,
    "C": 61.27079853477734
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27011170852929356,
    "p50_s_raw": 0.06005585426464677,
    "p95_s_adjusted": 0.30493060462176796,
    "p95_s_raw": 0.07746530231088398
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.92159889892,
      86.07279175908928
     ],
     "I": [
      51.12198637107415,
      61.99576687100331
     ],
     "capability": [
      66.53160497364678,
      73.32780427342016
     ],
     "composite": {
      "A": [
       63.84380091743296,
       68.17521491005895
      ],
      "B": [
       60.77468333880452,
       66.74957560085348
      ],
      "C": [
       46.32022510115811,
       68.17521491005895
      ]
     },
     "seed": 127
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.8928101186779,
     "capability": 71.18539464154858,
     "intelligence": 57.47797916441928
    },
    "per_type": {
     "choice": {
      "accuracy": 0.804,
      "calibration": 73.96093639513141,
      "cc": 74.93439936341332,
      "cc_by_type": {
       "choice": 74.93439936341332
      },
      "cc_tiered": 74.19503754391312,
      "cc_tiered_open": 74.77679085167708,
      "cc_tiered_sealed": 73.61328423614917,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.93439936341332
     },
     "noul": {
      "accuracy": 0.7493333333333333,
      "calibration": 91.9742721600182,
      "cc": 49.86666666666666,
      "cc_by_type": {
       "noul": 49.86666666666666
      },
      "cc_tiered": 44.48820081503986,
      "cc_tiered_open": 43.24154589371981,
      "cc_tiered_sealed": 45.734855736359904,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 49.86666666666666
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 88.7432218008841,
      "cc": 52.68777151445312,
      "cc_by_type": {
       "score": 52.68777151445312
      },
      "cc_tiered": 53.75069913430488,
      "cc_tiered_open": 54.96534000481365,
      "cc_tiered_sealed": 52.53605826379611,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.68777151445312
     }
    },
    "raw_on_A": {
     "calibration": 81.41272621760669,
     "capability": 70.66742916053838,
     "intelligence": 59.922132103470055
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 62.3292575088842,
    "Emphasis on Accuracy 60:20:20": 60.293685250997875,
    "Emphasis on Cost 20:20:60": 57.089082158875925,
    "Emphasis on Speed 20:60:20": 71.27827330005212,
    "Intelligence only": 57.477979164419274
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Foxl AI / The Bobcat Authors (admin@foxl.ai)",
   "axes": {
    "calibration": 80.53028652577869,
    "cost": 49.68577376542501,
    "intelligence": 65.56939608070681,
    "speed": 90.60003447236298
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.114937134190201,
      "ece_score": 77.01257316195979,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 76.19110571489888,
      "score_before_time_limit": 77.01257316195979,
      "time_limit_failed_share": 0.010666666666666666
     },
     "noul": {
      "abstention_rate": 0.10666666666666669,
      "brier": 0.08802311851821586,
      "ece": 0.07573463232351335,
      "n": 375,
      "score": 84.40052380977575,
      "score_before_time_limit": 84.85307353529733,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9066666666666666,
      "interval90_width": 0.1808888888888889,
      "n": 375,
      "nrps": 0.06948590052038994,
      "score": 80.99923005266146,
      "score_before_time_limit": 81.65244964986034,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.14873255324120163
     }
    },
    "score": 80.53028652577869
   },
   "capability": 73.04984130324274,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     65.16013011980922,
     68.48812233688949
    ],
    "B": [
     63.96437271803637,
     68.56858877197912
    ],
    "C": [
     64.92355836854192,
     68.48812233688949
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged. THIS IS THE REFERENCE, THE RATE AND THE LABEL THE PUBLISHED PAGE ALREADY CARRIES FOR THIS EXACT BASE on three rows published within the last day - xor-26b-a4b, surogate-rune-26b-a4b-v3-v16 and gev-26b-decide - so pricing this row any other way would make it inconsistent with the field it is placed in. RELEASE-LANE LINE, recorded and not acted on: pricing_v15.reference() RAISES on the author's exact declared spelling google/gemma-4-26B-A4B-it, because that string is in none of the three decision tables, while the lowercase google/gemma-4-26b-a4b-it IS in BASE_REFERENCES and in the dated snapshot at (0.09, 0.30). That is a CAPITALISATION gap - one notch tighter than run 56's -Base/instruct gap on Qwen3.5-35B-A3B - and the three published rows say in their own basis string that the frozen module has no explicit listing-decision line for this base. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: their build_response returns usage.output_tokens == 0 and the readout takes candidate logits at the first answer position of one forward pass, so this row's cost depends on the input rate alone and any other decision is reversible by arithmetic from the raw.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.15782652
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04754894380501015
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 13,
    "n_over_limit_answered_ok": 13,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bobcat Flash 1.2 (Gemma-4-26B-A4B-it + two merged rank-64 LoRA adapters, typed-decision readout at the first answer position, FP8, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights sanghwa-na/bobcat-flash-1.2@58d15456937ce58625b13a5e3d982846554a3aef, Apache-2.0, 51.68 GB BF16 - a merge of two rank-64 LoRA adapters on google/gemma-4-26B-A4B-it@4d7ae4984b7db7de8f8457170b3f1a419ee76d52 (0.6*A + 0.4*B, B being the Flash 1.1 adapter), file-by-file verified against the Hub's own manifest before the engine was started. Code github.com/foxl-ai/bobcat@43387a09bbc0310edcf57e2b21fa59ea81ac4166, Apache-2.0. NO MAPPING DECISION OF OURS EXISTS FOR THIS ROW, which is why it is worth saying so explicitly. The author implements TypeSafe's published System One contract, and their own src/bobcat/protocol.py parse_request accepts the JevBench canonical record EXACTLY as the UNCHANGED jevbench.adapters.typesafe already emits it: choice -> a dict of 1-255 named criteria, so the item's own labels are the keys and its own criteria[label] strings travel verbatim as the descriptions (their protocol renders {name, description} per option); score -> the item's ordered array of levels, which their own parse_request labels '0'..'n-1' - the same keys JevBench uses - with the item's own level text as each description, so the ordinal order is carried; noul -> criteria restricted to {true,false}, and the item's OWN true/false description strings are delivered (unlike run 56's APUS rows, whose runtime required canonical criteria there), with the answer returned as noul = P(yes). Their answer() refuses to reply unless the probabilities sum to one. Nothing was rewritten, no option was dropped, no item type was recorded as a miss, and no shim of ours sits anywhere in the path. STACK: the author's own published install line, verbatim from their model card's Quickstart - a uv-managed Python 3.12 (their note: it ships the headers Triton compiles against) and vllm==0.30.0 LETTING IT RESOLVE ITS OWN STACK, which gave torch 2.13.0+cu130, transformers 5.19.0, tokenizers 0.23.2. Served by THEIR OWN SERVER, bobcat.flash_server, with THEIR OWN published command verbatim: --engine vllm --quantization fp8 (their online dynamic FP8 of these BF16 weights, the served configuration) --temperature choice=0.594604,noul=0.353553,score=0.529732 --max-num-seqs 64 --max-model-len 98368 --schedule all --engine-arg max_num_batched_tokens=16384 --engine-arg attention_backend=TRITON_ATTN, with VLLM_USE_FLASHINFER_SAMPLER=0 as their command sets it. --local only because the server is on loopback on our own pod. One H100 PCIe 80 GB (Lium) - the second of the two cards the author measured. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. No file of theirs was edited and no prompt of ours exists. CALIBRATION IS THEIR PUBLISHED DEFAULT AND WAS NOT CHOSEN BY US. Their card ships two calibrations: the per-type triple used here, which is the Quickstart's - i.e. the published serve command of this release - and a single product temperature 0.9232 fitted by log-loss on their six product tasks. The triple was bound in PREREG-R57 s3.3 before any item was sent and the alternative was deliberately NOT probed on our items, because the author has ALREADY PUBLISHED both configurations' numbers on JevBench's public 231 (per-type Capability 68.2 = 57.5/78.9; product 68.3 = 54.3/82.3), so there was nothing to learn from our pool and choosing between them from our own measurement would have been exactly the mistake run 56's probe was designed to avoid. The author states the triple was fitted on a held-out split of 8,906 JevBench-FORMAT decisions (generated and public) for the highest JevBench-style Capability, and separately that no JevBench item was used for training, calibration, selection or prompt design, with generated rows audited against JevBench's public items at 0 exact and 0 8-gram overlaps. THE 23 LONG ITEMS: the server's limit is --max-model-len 98368 (their compiler gets max_model_len - 64) against a pool whose longest item is 128,111 BYTES of canonical record, and their RequestLimitError answers 422 with 'Inputs are never truncated' rather than truncating. This row therefore reads every item in full. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 57, benchmarkheaven.com/submit ref 7A714784 (bh-submit e9d576d7-e1f1-49d8-8151-05cdbbdd7ed4)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100 PCIe",
   "intelligence": {
    "I_open": 65.86976952005253,
    "I_sealed": 65.26902264136108,
    "base": 65.56939608070681,
    "excess": -1.9726407637467909,
    "gap": 0.6007468786914529,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.45043236055656,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 66.02705253224286,
       "judge": 80.22412656558997,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 66.6425120772947,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 56.516364122306314,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.947350572836605,
       "hard": 60.92245146041372,
       "judge": 57.42924259191816,
       "standard": 43.1193785164086
      }
     },
     "sealed|choice": {
      "cc": 75.92775512086644,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 78.71554208853745,
       "hard": 67.42081447963801,
       "judge": 81.3780260707635,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 59.790998006434116,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 37.86407766990292,
       "judge": 71.05263157894737,
       "standard": 74.64788732394365
      }
     },
     "sealed|score": {
      "cc": 60.088314796782676,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 85.14251727623912,
       "hard": 53.13372651343875,
       "judge": 54.946113599979476,
       "standard": 69.18369191894709
      }
     }
    }
   },
   "jevbench_score": 67.13750700906564,
   "key": "bobcat-flash-1.2",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "sanghwa-na/bobcat-flash-1.2@58d15456937ce58625b13a5e3d982846554a3aef",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.924924924924925,
    "decisive_rate": 0.8927613941018767,
    "supported": true,
    "valid": 373
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.17269878237248,
     "cost": 49.68577376542501,
     "intelligence": 65.81464576505633,
     "speed": 90.781726629608
    },
    "board_release": "v1.6.43",
    "capability": 73.4936722737144,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 14,
    "capability_rank": 30,
    "composite_ci95": {
     "A": [
      65.41706710923236,
      68.6674732651027
     ],
     "B": [
      64.1802436512738,
      68.73104866859511
     ],
     "C": [
      65.32359947723424,
      68.6674732651027
     ]
    },
    "jevbench_score": 67.34032787154446,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 23,
    "rank": 43,
    "ranks": {
     "A": 43,
     "B": 35,
     "C": 35,
     "capability": 54
    },
    "scores": {
     "A": 67.34032787154446,
     "B": 66.85676394909423,
     "C": 67.34032787154446
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2719114740379155,
     "p50_s_raw": 0.060955737018957734,
     "p95_s_adjusted": 0.3071848569670692,
     "p95_s_raw": 0.0785924284835346
    },
    "views": {
     "Balanced 33:33:33": 63.93311146268058,
     "Emphasis on Accuracy 60:20:20": 64.35172177912958,
     "Emphasis on Cost 20:20:60": 57.02049836458103,
     "Emphasis on Speed 20:60:20": 72.21838604704122,
     "Intelligence only": 64.99001759823823
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 48,
   "ranked": true,
   "ranks": {
    "A": 48,
    "B": 39,
    "C": 39,
    "capability": 69
   },
   "repo": "https://huggingface.co/sanghwa-na/bobcat-flash-1.2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.13750700906564,
    "B": 66.6455657929244,
    "C": 67.13750700906564
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2730665770126507,
    "p50_s_raw": 0.061533288506325334,
    "p95_s_adjusted": 0.3189539658022113,
    "p95_s_raw": 0.08447698290110565
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.59302008922947,
      83.34940918083149
     ],
     "I": [
      59.965147789446455,
      69.81159689545017
     ],
     "capability": [
      68.94776093285044,
      76.03279055626672
     ],
     "composite": {
      "A": [
       65.16013011980922,
       68.48812233688949
      ],
      "B": [
       63.96437271803637,
       68.56858877197912
      ],
      "C": [
       64.92355836854192,
       68.48812233688949
      ]
     },
     "seed": 24
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.53028652577869,
     "capability": 73.04984130324274,
     "intelligence": 65.56939608070681
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8106666666666666,
      "calibration": 76.19110571489888,
      "cc": 75.78697081363734,
      "cc_by_type": {
       "choice": 75.78697081363734
      },
      "cc_tiered": 75.1890937407115,
      "cc_tiered_open": 74.45043236055656,
      "cc_tiered_sealed": 75.92775512086644,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.78697081363734
     },
     "noul": {
      "accuracy": 0.8213333333333334,
      "calibration": 84.40052380977575,
      "cc": 64.26666666666667,
      "cc_by_type": {
       "noul": 64.26666666666667
      },
      "cc_tiered": 63.2167550418644,
      "cc_tiered_open": 66.6425120772947,
      "cc_tiered_sealed": 59.790998006434116,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.26666666666667
     },
     "score": {
      "accuracy": 0.6933333333333334,
      "calibration": 80.99923005266146,
      "cc": 61.138841659786316,
      "cc_by_type": {
       "score": 61.138841659786316
      },
      "cc_tiered": 58.3023394595445,
      "cc_tiered_open": 56.516364122306314,
      "cc_tiered_sealed": 60.088314796782676,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 61.138841659786316
     }
    },
    "raw_on_A": {
     "calibration": 76.02127380259917,
     "capability": 72.29741756972824,
     "intelligence": 68.57356133685731
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.010666666666666713,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.82440303033704,
    "Emphasis on Accuracy 60:20:20": 64.19059916552258,
    "Emphasis on Cost 20:20:60": 56.968574352651004,
    "Emphasis on Speed 20:60:20": 72.0885818239981,
    "Intelligence only": 64.74784078308473
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SPX AI team (SurdAI, formerly Simplex AI)",
   "axes": {
    "calibration": 90.20036269195566,
    "cost": 52.107175358413464,
    "intelligence": 58.43051843707937,
    "speed": 79.18845201110972
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05772702157730632,
      "ece_score": 88.45459568453875,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.45459568453875,
      "score_before_time_limit": 88.45459568453875,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18598382749326148,
      "brier": 0.08492257448407475,
      "ece": 0.04230529224114798,
      "n": 371,
      "score": 91.5389415517704,
      "score_before_time_limit": 91.5389415517704,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9322493224932249,
      "interval90_width": 0.30013550135501355,
      "n": 369,
      "nrps": 0.06769358396529002,
      "score": 90.60755083955786,
      "score_before_time_limit": 90.60755083955786,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06007769962177635
     }
    },
    "score": 90.20036269195566
   },
   "capability": 74.31544056451752,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": [
     64.32849489209721,
     67.89404426087012
    ],
    "B": [
     61.78442076552503,
     66.80330225899888
    ],
    "C": [
     50.560624091643184,
     67.89404426087012
    ]
   },
   "cost": {
    "basis": "Operator's own public-beta overage tariff, conservative reading USD 0.04 per 1M input tokens, output USD 0 (the provider's /v1/models payload reports output_price \"0\"): three dated primary readings of the operator's own published overage tariff: 24 Sep 2026 provider console (flash 0.025, pro 0.08 per 1M input), 1 Oct 2026 docs page (flash 0.04, pro 0.02), 6 Oct 2026 docs page AND /v1/models payload agreeing (flash 0.02, pro 0.04). Sources conflict, so the per-model MAXIMUM is used, which is the published fastino-gliner-2-5-decide precedent (\"published sources conflict ... conservative higher\"). Today's agreeing pair would be USD 0.02/M, i.e. half this row's cost. No base-model floor applies: the submitter states the base model stays sealed until a later open release, so no base model exists to reference. Priced on usage.input_tokens as the provider reports them. The provider also reports billable_input_tokens = input_tokens / 2 on every single item (297/297 of the public set, exactly 2.0): its default effort 2 renders the prompt twice and its docs bundle the second pass free during public beta (\"default effort 2 is included at the single-pass input price\"). That bundle is a public-beta promotion, and TASK.md forbids a free or promotional tier setting the price, so the two passes actually consumed are charged. The operator's own billed figure is exactly half of this row's cost",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.03887914666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.039484576844955994
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SPX-CD Flash (SurdAI, hosted /v1/systemone, Oct-4 checkpoint)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows) under Florian's 5 Oct 2026 full-set amendment, through the release lane's own full-set API lane (run_api_full_v161.py, rotation.py expose --full-set logged 1,200 sealed items to provider 'surd' before the first request); stock jevbench.adapters.typesafe with an endpoint override only, reasoning at the provider default, one request at a time, no retries; add-requests run 53, benchmarkheaven.com/submit ref 3FF00545",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 59.99669320470862,
    "I_sealed": 56.86434366945013,
    "base": 58.43051843707937,
    "excess": 0.5589618928202476,
    "gap": 3.1323495352584914,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.41749935055667,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396666,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 52.89855072463768,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 0.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 51.67402953893152,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.86207439440385,
       "hard": 52.93207079861204,
       "judge": 54.720404668190085,
       "standard": 42.994361897946476
      }
     },
     "sealed|choice": {
      "cc": 71.02357528674706,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 67.92986425339366,
       "judge": 68.34264432029796,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 45.34763039144423,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 20.388349514563096,
       "judge": 52.63157894736843,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 54.221825330159106,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 80.5246722641586,
       "hard": 48.238601087091794,
       "judge": 55.5539283927005,
       "standard": 51.038695755481854
      }
     }
    }
   },
   "jevbench_score": 66.64434875110587,
   "key": "spx-cd-flash",
   "last_measured_on": "2026-10-06",
   "licence": null,
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "provider-reported measurement model slug: spx-cd-flash",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9370860927152318,
    "decisive_rate": 0.8140161725067385,
    "supported": true,
    "valid": 371
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 21,
    "axes": {
     "calibration": 90.20036269195566,
     "cost": 52.107175358413464,
     "intelligence": 58.43051843707937,
     "speed": 79.39117060033084
    },
    "board_release": "v1.6.43",
    "capability": 74.31544056451752,
    "capability_api_board_rank": 12,
    "capability_open_board_rank": null,
    "capability_rank": 26,
    "composite_ci95": {
     "A": [
      64.36187068361473,
      67.93122351228547
     ],
     "B": [
      61.80904827802077,
      66.83209430434223
     ],
     "C": [
      50.5869412571725,
      67.93122351228547
     ]
    },
    "jevbench_score": 66.68017155386212,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 44,
    "ranks": {
     "A": 44,
     "B": 43,
     "C": 38,
     "capability": 48
    },
    "scores": {
     "A": 66.68017155386212,
     "B": 64.84899920515487,
     "C": 63.23735362072274
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.9804960675537586,
     "p50_s_raw": 0.9804960675537586,
     "p95_s_adjusted": 1.1733756832778452,
     "p95_s_raw": 1.1733756832778452
    },
    "views": {
     "Balanced 33:33:33": 61.34791411142523,
     "Emphasis on Accuracy 60:20:20": 60.146681111986254,
     "Emphasis on Cost 20:20:60": 57.28436724874348,
     "Emphasis on Speed 20:60:20": 67.48263394824296,
     "Intelligence only": 58.43051843707938
    }
   },
   "provenance": {
    "carried_metadata_from": "Current-pool API measurement registry",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 50,
   "ranked": true,
   "ranks": {
    "A": 50,
    "B": 46,
    "C": 41,
    "capability": 59
   },
   "repo": "https://surd.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.64434875110587,
    "B": 64.82189014379784,
    "C": 63.20338041410385
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.0145484656095505,
    "p50_s_raw": 1.0145484656095505,
    "p95_s_adjusted": 1.18817935064435,
    "p95_s_raw": 1.18817935064435
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.82360765229315,
      91.10972764372464
     ],
     "I": [
      53.013471055498634,
      62.79232050259918
     ],
     "capability": [
      69.82626986507542,
      76.43255727254187
     ],
     "composite": {
      "A": [
       64.32849489209721,
       67.89404426087012
      ],
      "B": [
       61.78442076552503,
       66.80330225899888
      ],
      "C": [
       50.560624091643184,
       67.89404426087012
      ]
     },
     "seed": 136
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.20036269195566,
     "capability": 74.31544056451752,
     "intelligence": 58.43051843707937
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7893333333333333,
      "calibration": 88.45459568453875,
      "cc": 73.05874217292042,
      "cc_by_type": {
       "choice": 73.05874217292042
      },
      "cc_tiered": 73.22053731865186,
      "cc_tiered_open": 75.41749935055667,
      "cc_tiered_sealed": 71.02357528674706,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.05874217292042
     },
     "noul": {
      "accuracy": 0.7546666666666667,
      "calibration": 91.5389415517704,
      "cc": 50.933333333333344,
      "cc_by_type": {
       "noul": 50.933333333333344
      },
      "cc_tiered": 49.12309055804096,
      "cc_tiered_open": 52.89855072463768,
      "cc_tiered_sealed": 45.34763039144423,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 50.933333333333344
     },
     "score": {
      "accuracy": 0.6746666666666666,
      "calibration": 90.60755083955786,
      "cc": 54.63501832116133,
      "cc_by_type": {
       "score": 54.63501832116133
      },
      "cc_tiered": 52.94792743454531,
      "cc_tiered_open": 51.67402953893152,
      "cc_tiered_sealed": 54.221825330159106,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.63501832116133
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 61.307488965987744,
    "Emphasis on Accuracy 60:20:20": 60.12336025933381,
    "Emphasis on Cost 20:20:60": 57.26321281799819,
    "Emphasis on Speed 20:60:20": 67.39464498875742,
    "Intelligence only": 58.43051843707938
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 82.34718405098182,
    "cost": 50.78502198104071,
    "intelligence": 58.54290392078144,
    "speed": 85.76268320171138
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13017175324675329,
      "ece_score": 73.96564935064934,
      "mean_tvd": null,
      "n": 738,
      "n_ece_hard": 308,
      "n_tvd": 0,
      "score": 73.37392415584415,
      "score_before_time_limit": 73.96564935064934,
      "time_limit_failed_share": 0.008
     },
     "noul": {
      "abstention_rate": 0.1967654986522911,
      "brier": 0.06013664803234495,
      "ece": 0.0693757412398923,
      "n": 371,
      "score": 86.12485175202154,
      "score_before_time_limit": 86.12485175202154,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.640469738030713,
      "n": 369,
      "nrps": 0.06538499443315268,
      "score": 87.54277624507979,
      "score_before_time_limit": 87.77684783931798,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.0895390243902439
     }
    },
    "score": 82.34718405098182
   },
   "capability": 70.44504398588163,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     63.76759901152183,
     66.95638771586961
    ],
    "B": [
     61.3751450873436,
     66.03818487950825
    ],
    "C": [
     49.99900628994275,
     66.95638771586961
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it, the reference of the published deck31b and quyet-1-0-large rows, USD 0.09 per 1M input and USD 0.34 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04500444
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04370179417738659
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 7,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider chat on Gemma-4-31B-it (Mapika, frozen base, inference technique)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 55.72975554572549,
    "I_sealed": 61.356052295837394,
    "base": 58.54290392078144,
    "excess": -8.199684392550148,
    "gap": -5.626296750111905,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 76.96981695560446,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 43.846790890269155,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 46.372658791302854,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.266060606060606,
       "hard": 49.83425925925926,
       "judge": 51.426101694915246,
       "standard": 28.422592592592586
      }
     },
     "sealed|choice": {
      "cc": 82.26456829463379,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 90.68804966373514,
       "hard": 73.02036199095022,
       "judge": 88.8268156424581,
       "standard": 86.69786919571378
      }
     },
     "sealed|noul": {
      "cc": 54.93243610873211,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 37.86407766990292,
       "judge": 63.1578947368421,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 46.871152484146286,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 60.35531914893617,
       "hard": 42.373504901960835,
       "judge": 52.318562874251484,
       "standard": 40.95324873096443
      }
     }
    }
   },
   "jevbench_score": 66.03261747636077,
   "key": "decider-chat-gemma4-31b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475 + Mapika/decider-chat-gemma4-31b@22975cbd1716170264a1d2a7e1105787e08c5d95",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9765100671140939,
    "decisive_rate": 0.8032345013477089,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.62244964732962,
     "cost": 50.78502198104071,
     "intelligence": 58.54290392078144,
     "speed": 85.78163968405963
    },
    "board_release": "v1.6.43",
    "capability": 70.58267678405554,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 25,
    "capability_rank": 44,
    "composite_ci95": {
     "A": [
      63.831274751690664,
      67.00563695142573
     ],
     "B": [
      61.40092144952185,
      66.08555760610543
     ],
     "C": [
      50.02666080334015,
      67.00563695142573
     ]
    },
    "jevbench_score": 66.07956202755186,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 25,
    "rank": 46,
    "ranks": {
     "A": 46,
     "B": 44,
     "C": 39,
     "capability": 75
    },
    "scores": {
     "A": 66.07956202755186,
     "B": 64.42088721187244,
     "C": 62.90905728971202
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4011345919920132,
     "p50_s_raw": 0.12556729599600658,
     "p95_s_adjusted": 0.658485045604175,
     "p95_s_raw": 0.2542425228020875
    },
    "views": {
     "Balanced 33:33:33": 61.94527894229558,
     "Emphasis on Accuracy 60:20:20": 60.537949991324034,
     "Emphasis on Cost 20:20:60": 56.94013011167525,
     "Emphasis on Speed 20:60:20": 69.69141016272579,
     "Intelligence only": 58.54290392078144
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 52,
   "ranked": true,
   "ranks": {
    "A": 52,
    "B": 50,
    "C": 42,
    "capability": 89
   },
   "repo": "https://github.com/Mapika/decider",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.03261747636077,
    "B": 64.38518770823907,
    "C": 62.864365143309946
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.40184165698010477,
    "p50_s_raw": 0.12592082849005237,
    "p95_s_adjusted": 0.6602018301025965,
    "p95_s_raw": 0.2551009150512982
   },
   "status": {
    "answered_ok": 1478,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.49144818930593,
      82.08290139197796
     ],
     "I": [
      53.001527166194506,
      62.9130738579859
     ],
     "capability": [
      66.47453104990954,
      71.637319105497
     ],
     "composite": {
      "A": [
       63.76759901152183,
       66.95638771586961
      ],
      "B": [
       61.3751450873436,
       66.03818487950825
      ],
      "C": [
       49.99900628994275,
       66.95638771586961
      ]
     },
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.34718405098182,
     "capability": 70.44504398588163,
     "intelligence": 58.54290392078144
    },
    "per_type": {
     "choice": {
      "accuracy": 0.856,
      "calibration": 73.37392415584415,
      "cc": 81.5844566751608,
      "cc_by_type": {
       "choice": 81.5844566751608
      },
      "cc_tiered": 79.61719262511912,
      "cc_tiered_open": 76.96981695560446,
      "cc_tiered_sealed": 82.26456829463379,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.5844566751608
     },
     "noul": {
      "accuracy": 0.776,
      "calibration": 86.12485175202154,
      "cc": 55.2,
      "cc_by_type": {
       "noul": 55.2
      },
      "cc_tiered": 49.38961349950063,
      "cc_tiered_open": 43.846790890269155,
      "cc_tiered_sealed": 54.93243610873211,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 55.2
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 87.54277624507979,
      "cc": 46.969219298245605,
      "cc_by_type": {
       "score": 46.969219298245605
      },
      "cc_tiered": 46.621905637724566,
      "cc_tiered_open": 46.372658791302854,
      "cc_tiered_sealed": 46.871152484146286,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 46.969219298245605
     }
    },
    "raw_on_A": {
     "calibration": 80.29664856797598,
     "capability": 70.99028048840852,
     "intelligence": 61.68391240884105
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 61.94198332242641,
    "Emphasis on Accuracy 60:20:20": 60.53606140467672,
    "Emphasis on Cost 20:20:60": 56.93845933189856,
    "Emphasis on Speed 20:60:20": 69.68390209793014,
    "Intelligence only": 58.54290392078144
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "HopitAI (@ArchieIndian)",
   "axes": {
    "calibration": 84.63452107883388,
    "cost": 56.15749203571245,
    "intelligence": 51.6576814732076,
    "speed": 83.63977704006717
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11381338060621848,
      "ece_score": 77.2373238787563,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 77.2373238787563,
      "score_before_time_limit": 77.2373238787563,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2075471698113207,
      "brier": 0.09205588963446744,
      "ece": 0.056376852598614385,
      "n": 371,
      "score": 88.72462948027713,
      "score_before_time_limit": 88.72462948027713,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.40492321589882563,
      "n": 369,
      "nrps": 0.08202603571136496,
      "score": 87.94160987746824,
      "score_before_time_limit": 87.94160987746824,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0795708833696351
     }
    },
    "score": 84.63452107883388
   },
   "capability": 68.14610127602074,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     54.559416041823084,
     67.28202466694631
    ],
    "B": [
     50.95653451796613,
     64.6241123438328
    ],
    "C": [
     37.88848336237714,
     58.47419624222569
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist. The reference is the one the rows PUBLISHED on this same base use - cygnet, jev-omni and winnow-12b - OpenRouter google/gemma-3-12b-it, the nearest publicly hosted 12B Gemma sibling, at its FULL catalog pair USD 0.05 per 1M input and USD 0.15 per 1M output. PREREG-R54-AMENDMENT-1 uses the full pair rather than run 53's output=0, so a row that emits tokens pays for them; where a row generates nothing the output term is zero anyway. x this row's own measured input/output tokens, the deck31b method. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.028490900000000006
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02893456330399459
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Hopper 12B trained (gemma-4-12B-it frozen + LoRA r32 unmerged, one forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. Serving code github.com/hopit-ai/hopper tag 12b-1.1.1, installed with the author's own one-line pip install; adapter HopitAI/hopper@4b605f25eb903c2cf4d121040ffb097240b3bc37 (branch hopper-12b), LoRA r32 alpha 64, loaded UNMERGED because the author states a bf16 merge changes answers; base google/gemma-4-12B-it@707f0a3b, frozen; readout map maps/hopper-12b-trained-readout.json as shipped in their package. 1.1.1 is their own fix for corrupted option scores at prompt lengths one above a multiple of 32. Their flags left at their defaults: --shortlist tournament (their measured default) for menus over 26 options, --cuda-graphs OFF. The author asked on 6 Oct 2026 that the TRAINED version replace the untrained Hopper 12B request in GitHub #164; this is that row. NOT the published `hopper` row (HopitAI, rank 25), a different system. One RTX 6000 Ada 48 GB (Lium, Toronto); the author ran theirs on an L40S. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 54, GitHub #164",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 6000 Ada",
   "intelligence": {
    "I_open": 52.49257010742919,
    "I_sealed": 50.822792838986025,
    "base": 51.6576814732076,
    "excess": -0.9036103739950812,
    "gap": 1.6697772684431627,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 68.64489681634264,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 60.36489462094999,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 43.547273982056595,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 45.285539523888346,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.928801818275204,
       "hard": 41.02383720746134,
       "judge": 39.409476186258296,
       "standard": 53.30140801599398
      }
     },
     "sealed|choice": {
      "cc": 65.07976853625175,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 82.70637794693668,
       "hard": 58.2579185520362,
       "judge": 62.756052141527014,
       "standard": 73.39573839142753
      }
     },
     "sealed|noul": {
      "cc": 41.18694322295551,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 18.446601941747566,
       "judge": 39.47368421052631,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 46.20166675775083,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.66271785418321,
       "hard": 37.07866936088858,
       "judge": 45.777923536915246,
       "standard": 54.3527508345125
      }
     }
    }
   },
   "jevbench_score": 65.64176708798277,
   "key": "hopper-12b-trained",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "HopitAI/hopper@4b605f25eb903c2cf4d121040ffb097240b3bc37",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9319727891156463,
    "decisive_rate": 0.7924528301886793,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.63452107883388,
     "cost": 56.15749203571245,
     "intelligence": 51.6576814732076,
     "speed": 84.57301814857726
    },
    "board_release": "v1.6.43",
    "capability": 68.14610127602074,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 31,
    "capability_rank": 52,
    "composite_ci95": {
     "A": [
      54.67384317203267,
      67.43166618883266
     ],
     "B": [
      51.03564669331923,
      64.73449720897486
     ],
     "C": [
      37.9679466472449,
      58.605012542436775
     ]
    },
    "jevbench_score": 65.78419365065008,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 26,
    "rank": 47,
    "ranks": {
     "A": 47,
     "B": 49,
     "C": 54,
     "capability": 89
    },
    "scores": {
     "A": 65.78419365065008,
     "B": 62.37284887464997,
     "C": 48.76282692619866
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4776301250443794,
     "p50_s_raw": 0.1638150625221897,
     "p95_s_adjusted": 0.7304768792411778,
     "p95_s_raw": 0.2902384396205889
    },
    "views": {
     "Balanced 33:33:33": 61.23777421409552,
     "Emphasis on Accuracy 60:20:20": 57.0087849790455,
     "Emphasis on Cost 20:20:60": 59.09921236284679,
     "Emphasis on Speed 20:60:20": 68.83489755597157,
     "Intelligence only": 51.6576814732076
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 53,
   "ranked": true,
   "ranks": {
    "A": 53,
    "B": 55,
    "C": 61,
    "capability": 104
   },
   "repo": "https://huggingface.co/HopitAI/hopper",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 65.64176708798277,
    "B": 62.27036485482303,
    "C": 48.6572526014312
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5167318890220486,
    "p50_s_raw": 0.18336594451102428,
    "p95_s_adjusted": 0.8370608537457883,
    "p95_s_raw": 0.34353042687289415
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.81171527967304,
      87.00046880053556
     ],
     "I": [
      46.53255595156734,
      55.86954644502632
     ],
     "capability": [
      64.05753668009696,
      70.56574900316099
     ],
     "composite": {
      "A": [
       54.559416041823084,
       67.28202466694631
      ],
      "B": [
       50.95653451796613,
       64.6241123438328
      ],
      "C": [
       37.88848336237714,
       58.47419624222569
      ]
     },
     "seed": 59
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.63452107883388,
     "capability": 68.14610127602074,
     "intelligence": 51.6576814732076
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7413333333333333,
      "calibration": 77.2373238787563,
      "cc": 66.92022773130735,
      "cc_by_type": {
       "choice": 66.92022773130735
      },
      "cc_tiered": 66.8623326762972,
      "cc_tiered_open": 68.64489681634264,
      "cc_tiered_sealed": 65.07976853625175,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.92022773130735
     },
     "noul": {
      "accuracy": 0.7306666666666667,
      "calibration": 88.72462948027713,
      "cc": 46.13333333333334,
      "cc_by_type": {
       "noul": 46.13333333333334
      },
      "cc_tiered": 42.36710860250605,
      "cc_tiered_open": 43.547273982056595,
      "cc_tiered_sealed": 41.18694322295551,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.13333333333334
     },
     "score": {
      "accuracy": 0.5973333333333334,
      "calibration": 87.94160987746824,
      "cc": 46.77672583471599,
      "cc_by_type": {
       "score": 46.77672583471599
      },
      "cc_tiered": 45.74360314081959,
      "cc_tiered_open": 45.285539523888346,
      "cc_tiered_sealed": 46.20166675775083,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 46.77672583471599
     }
    },
    "raw_on_A": {
     "calibration": 82.16875295682286,
     "capability": 68.22397555466662,
     "intelligence": 54.279198152510396
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 61.07329966968431,
    "Emphasis on Accuracy 60:20:20": 56.92315813595121,
    "Emphasis on Cost 20:20:60": 59.00719583746461,
    "Emphasis on Speed 20:60:20": 68.46185553825565,
    "Intelligence only": 51.6576814732076
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Surogate",
   "axes": {
    "calibration": 90.88042351612289,
    "cost": 49.597326937281316,
    "intelligence": 56.055735926690254,
    "speed": 86.53673235812263
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05932792591743529,
      "ece_score": 88.13441481651294,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.13441481651294,
      "score_before_time_limit": 88.13441481651294,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2695417789757413,
      "brier": 0.08438373400978966,
      "ece": 0.039029372562851536,
      "n": 371,
      "score": 92.19412548742969,
      "score_before_time_limit": 92.19412548742969,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967479674796748,
      "interval90_width": 0.37466124661246614,
      "n": 369,
      "nrps": 0.0647012985012109,
      "score": 92.31273024442605,
      "score_before_time_limit": 92.31273024442605,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04452204830513414
     }
    },
    "score": 90.88042351612289
   },
   "capability": 73.46807972140657,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     62.77223171595613,
     66.12858031750243
    ],
    "B": [
     59.73287020382695,
     64.65813084240443
    ],
    "C": [
     45.12798820443921,
     66.0889416014453
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, the M2 reference of the published surogate-rune-26b-a4b-v3 row (the frozen module has no explicit listing-decision line for this base: release-lane line), USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04713878
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.047872830060934325
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Surogate Rune 26B-A4B v3 (v1.6 pool)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 58.17716696493766,
    "I_sealed": 53.934304888442846,
    "base": 56.055735926690254,
    "excess": 1.6694744340565726,
    "gap": 4.242862076494816,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.95733901606096,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 66.02705253224286,
       "judge": 80.22412656558997,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 42.59075224292616,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 19.999999999999996,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 55.98340963582588,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.28084236876889,
       "hard": 59.46587194956875,
       "judge": 61.146761005919444,
       "standard": 38.12474158672826
      }
     },
     "sealed|choice": {
      "cc": 73.06247198507991,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 68.94796380090497,
       "judge": 73.9292364990689,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 34.31372825609765,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 16.50485436893203,
       "judge": 36.8421052631579,
       "standard": 49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 54.426714424150994,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 71.74170476496207,
       "hard": 48.52758483569058,
       "judge": 67.30550244555229,
       "standard": 38.24929639856435
      }
     }
    }
   },
   "jevbench_score": 64.98952113905437,
   "key": "surogate-rune-26b-a4b-v3-v16",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "surogate/rune-26b-a4b-GGUF@bd4a7cbbed66af1a95dfbac12cabf52e6e677411",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.966789667896679,
    "decisive_rate": 0.7304582210242587,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.88042351612289,
     "cost": 49.597326937281316,
     "intelligence": 56.055735926690254,
     "speed": 86.95393114544444
    },
    "board_release": "v1.6.43",
    "capability": 73.46807972140657,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 15,
    "capability_rank": 31,
    "composite_ci95": {
     "A": [
      62.82778837763545,
      66.19023981114698
     ],
     "B": [
      59.77310724201828,
      64.70527953464652
     ],
     "C": [
      45.16787327358027,
      66.15052716337196
     ]
    },
    "jevbench_score": 65.0490738113794,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 27,
    "rank": 48,
    "ranks": {
     "A": 48,
     "B": 47,
     "C": 45,
     "capability": 55
    },
    "scores": {
     "A": 65.0490738113794,
     "B": 62.796501415524304,
     "C": 56.77782262023472
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3823961119982414,
     "p50_s_raw": 0.1161980559991207,
     "p95_s_adjusted": 0.5273432816902641,
     "p95_s_raw": 0.18867164084513205
    },
    "views": {
     "Balanced 33:33:33": 59.63127588509936,
     "Emphasis on Accuracy 60:20:20": 57.75697016597956,
     "Emphasis on Cost 20:20:60": 54.76969389057124,
     "Emphasis on Speed 20:60:20": 67.8565544623977,
     "Intelligence only": 55.15648620822009
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card metadata",
    "kind": "v1.6.1 addendum a7 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 55,
   "ranked": true,
   "ranks": {
    "A": 55,
    "B": 53,
    "C": 49,
    "capability": 65
   },
   "repo": "https://huggingface.co/surogate/rune-26b-a4b-GGUF",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 64.98952113905437,
    "B": 62.75209247137393,
    "C": 56.7258423095598
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3835699379909784,
    "p50_s_raw": 0.11678496899548918,
    "p95_s_adjusted": 0.5787382701702881,
    "p95_s_raw": 0.21436913508514407
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.71263475548702,
      91.20016913118003
     ],
     "I": [
      50.88578039677805,
      60.44466553566163
     ],
     "capability": [
      69.30942731571565,
      74.85354775896664
     ],
     "composite": {
      "A": [
       62.77223171595613,
       66.12858031750243
      ],
      "B": [
       59.73287020382695,
       64.65813084240443
      ],
      "C": [
       45.12798820443921,
       66.0889416014453
      ]
     },
     "seed": 116
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.88042351612289,
     "capability": 73.46807972140657,
     "intelligence": 56.055735926690254
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8,
      "calibration": 88.13441481651294,
      "cc": 74.4228564932789,
      "cc_by_type": {
       "choice": 74.4228564932789
      },
      "cc_tiered": 74.50990550057043,
      "cc_tiered_open": 75.95733901606096,
      "cc_tiered_sealed": 73.06247198507991,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.4228564932789
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 92.19412548742969,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 38.4522402495119,
      "cc_tiered_open": 42.59075224292616,
      "cc_tiered_sealed": 34.31372825609765,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.6666666666666666,
      "calibration": 92.31273024442605,
      "cc": 54.21453874970139,
      "cc_by_type": {
       "score": 54.21453874970139
      },
      "cc_tiered": 55.20506202998844,
      "cc_tiered_open": 55.98340963582588,
      "cc_tiered_sealed": 54.426714424150994,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.21453874970139
     }
    },
    "raw_on_A": {
     "calibration": 89.39331463026028,
     "capability": 74.65140006965763,
     "intelligence": 59.909485509054974
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 59.564561791551476,
    "Emphasis on Accuracy 60:20:20": 57.71940087452699,
    "Emphasis on Cost 20:20:60": 54.73590923952626,
    "Emphasis on Speed 20:60:20": 67.70123836898624,
    "Intelligence only": 55.15648620822009
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AutoTrust AI Lab",
   "axes": {
    "calibration": 90.33483359631919,
    "cost": 51.12030271996105,
    "intelligence": 49.817428608862244,
    "speed": 89.61965681094298
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05524139109810591,
      "ece_score": 88.95172178037882,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.95172178037882,
      "score_before_time_limit": 88.95172178037882,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.30188679245283023,
      "brier": 0.091757346419259,
      "ece": 0.03659159632329119,
      "n": 371,
      "score": 92.68168073534177,
      "score_before_time_limit": 92.68168073534177,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.4146341463414633,
      "n": 369,
      "nrps": 0.07505075554087659,
      "score": 89.37109827323698,
      "score_before_time_limit": 89.37109827323698,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06876363949719191
     }
    },
    "score": 90.33483359631919
   },
   "capability": 70.07613110259072,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     48.09860005910896,
     66.25494220668105
    ],
    "B": [
     44.52100692860825,
     63.42030776707453
    ],
    "C": [
     33.401805596603445,
     54.41354624395857
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, the M2 reference of the published surogate-rune-26b-a4b-v3 row (the frozen module has no explicit listing-decision line for this base: release-lane line), USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04193846
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04259153012863913
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "GEV-26B-Decide (AutoTrust, Gemma-4-26B-A4B + LoRA + head)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 51.11414911073161,
    "I_sealed": 48.520708106992885,
    "base": 49.817428608862244,
    "excess": 0.02005336130048141,
    "gap": 2.593441003738725,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.91276210170015,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 69.80182447310474,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 35.146307798481715,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 47.28337743201297,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.41806192777556,
       "hard": 49.795958510853765,
       "judge": 44.087055938179034,
       "standard": 39.48535526720095
      }
     },
     "sealed|choice": {
      "cc": 69.65586334909833,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 64.36651583710406,
       "judge": 69.27374301675978,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 27.311750523580937,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 8.737864077669899,
       "judge": 28.947368421052634,
       "standard": 43.66197183098593
      }
     },
     "sealed|score": {
      "cc": 48.59451044829939,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.07336665308927,
       "hard": 39.678983511563985,
       "judge": 58.83673780283293,
       "standard": 41.82279518757497
      }
     }
    }
   },
   "jevbench_score": 64.18747531452154,
   "key": "gev-26b-decide",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "autotrust/GEV-26B-Decide@7c89590ead085bf77630b4bf68264ea30b6ddc78",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9613899613899614,
    "decisive_rate": 0.6981132075471698,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.33483359631919,
     "cost": 51.12030271996105,
     "intelligence": 49.817428608862244,
     "speed": 89.87310530072351
    },
    "board_release": "v1.6.43",
    "capability": 70.07613110259072,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 26,
    "capability_rank": 46,
    "composite_ci95": {
     "A": [
      48.122003834420376,
      66.28949313101391
     ],
     "B": [
      44.53704625593728,
      63.445630864100984
     ],
     "C": [
      33.418058218347475,
      54.44178965547935
     ]
    },
    "jevbench_score": 64.2201412904201,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 29,
    "rank": 50,
    "ranks": {
     "A": 50,
     "B": 50,
     "C": 58,
     "capability": 78
    },
    "scores": {
     "A": 64.2201412904201,
     "B": 60.60132222808652,
     "C": 44.59732034056951
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2535779389785603,
     "p50_s_raw": 0.051788969489280134,
     "p95_s_adjusted": 0.40604858579754366,
     "p95_s_raw": 0.12802429289877182
    },
    "views": {
     "Balanced 33:33:33": 58.6687573959346,
     "Emphasis on Accuracy 60:20:20": 54.59949882434838,
     "Emphasis on Cost 20:20:60": 55.22102328110309,
     "Emphasis on Speed 20:60:20": 67.97953354893902,
     "Intelligence only": 49.45428333114776
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 56,
   "ranked": true,
   "ranks": {
    "A": 56,
    "B": 56,
    "C": 66,
    "capability": 92
   },
   "repo": "https://huggingface.co/autotrust/GEV-26B-Decide",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 64.18747531452154,
    "B": 60.578048741267985,
    "C": 44.5746356350844
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.258882079977775,
    "p50_s_raw": 0.054441039988887496,
    "p95_s_adjusted": 0.4216308011615183,
    "p95_s_raw": 0.13581540058075917
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.06256442860612,
      90.45126283775208
     ],
     "I": [
      44.10305960817391,
      54.39355915396382
     ],
     "capability": [
      65.77131123789238,
      71.56937054169023
     ],
     "composite": {
      "A": [
       48.09860005910896,
       66.25494220668105
      ],
      "B": [
       44.52100692860825,
       63.42030776707453
      ],
      "C": [
       33.401805596603445,
       54.41354624395857
      ]
     },
     "seed": 42
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.33483359631919,
     "capability": 70.07613110259072,
     "intelligence": 49.817428608862244
    },
    "per_type": {
     "choice": {
      "accuracy": 0.772,
      "calibration": 88.95172178037882,
      "cc": 70.84205640233793,
      "cc_by_type": {
       "choice": 70.84205640233793
      },
      "cc_tiered": 70.28431272539925,
      "cc_tiered_open": 70.91276210170015,
      "cc_tiered_sealed": 69.65586334909833,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.84205640233793
     },
     "noul": {
      "accuracy": 0.664,
      "calibration": 92.68168073534177,
      "cc": 32.800000000000004,
      "cc_by_type": {
       "noul": 32.800000000000004
      },
      "cc_tiered": 31.229029161031328,
      "cc_tiered_open": 35.146307798481715,
      "cc_tiered_sealed": 27.311750523580937,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.800000000000004
     },
     "score": {
      "accuracy": 0.656,
      "calibration": 89.37109827323698,
      "cc": 48.08438271156575,
      "cc_by_type": {
       "score": 48.08438271156575
      },
      "cc_tiered": 47.93894394015618,
      "cc_tiered_open": 47.28337743201297,
      "cc_tiered_sealed": 48.59451044829939,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.08438271156575
     }
    },
    "raw_on_A": {
     "calibration": 88.68291663446959,
     "capability": 72.10556374473956,
     "intelligence": 55.528210855009505
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 58.632411325100584,
    "Emphasis on Accuracy 60:20:20": 54.58060625238284,
    "Emphasis on Cost 20:20:60": 55.20169821615017,
    "Emphasis on Speed 20:60:20": 67.89175658522586,
    "Intelligence only": 49.45428333114776
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "SPX AI team (SurdAI)",
   "axes": {
    "calibration": 87.47801599949784,
    "cost": 58.27072766098771,
    "intelligence": 48.779308810848946,
    "speed": 89.06649387039909
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04837927211703187,
      "ece_score": 90.32414557659362,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 90.32414557659362,
      "score_before_time_limit": 90.32414557659362,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3207547169811321,
      "brier": 0.10397941406905582,
      "ece": 0.08441105054816544,
      "n": 371,
      "score": 83.11778989036691,
      "score_before_time_limit": 83.11778989036691,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.975609756097561,
      "interval90_width": 0.31187895212285444,
      "n": 369,
      "nrps": 0.06867686607554577,
      "score": 88.99211253153302,
      "score_before_time_limit": 88.99211253153302,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07574044164689696
     }
    },
    "score": 87.47801599949784
   },
   "capability": 68.1286624051734,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     48.97252601630862,
     68.51906187054689
    ],
    "B": [
     44.91858495976889,
     64.99516705077025
    ],
    "C": [
     34.00869862243654,
     55.01698761837606
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist. The reference is the one the three rows PUBLISHED on this same base all use - cygnet, jev-omni and winnow-12b - OpenRouter google/gemma-3-12b-it list price USD 0.05 per 1M input, USD 0.00 per 1M output (the nearest publicly hosted 12B Gemma sibling; output is 0 because the runner reads candidate logits in one forward pass and generates nothing), x this row's own measured usage.input_tokens, the deck31b method. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.024225066666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.024602301963439405
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SPX-CD-Omni (SurdAI, google/gemma-4-12B-it + LoRA r32, one forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows, 1,477 ok; the 23 refusals are exactly the 23 long-context items, through the author's own \"Prompt exceeds context limit; no automatic truncation\" guard at their shipped --max-context 16384, although the base model itself declares 262,144 positions). Author's own code unmodified (infer.py / decision.py, sha256 1c5db5de.../43389f44...) with their shipped --backend transformers on the unquantized base and the flags their own EVALUATION.md prescribes FOR JEVBENCH: --prompt-format open-format --effort 1 --temperature 1.0. Their requirements-transformers.txt is missing torchvision, which transformers 5.16.1 needs for the Gemma 4 unified processor; adding it was the only deviation and it changes no numerics. One RTX PRO 6000 Blackwell (Lium, Brasov, Romania), torch 2.12.0+cu130, transformers 5.16.1, peft 0.20.0. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment; raw p50/p95 are in the run 53 receipts; add-requests run 53, benchmarkheaven.com/submit ref 75D5EF0C",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000 Blackwell",
   "intelligence": {
    "I_open": 47.456697185695944,
    "I_sealed": 50.10192043600195,
    "base": 48.779308810848946,
    "excess": -5.2186108927442465,
    "gap": -2.645223250306003,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 69.72620969563555,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 69.80182447310474,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 16.937888198757765,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 55.70599366269451,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 45.44648364886738,
       "hard": 61.913039440117416,
       "judge": 59.403373958799065,
       "standard": 42.875586670605394
      }
     },
     "sealed|choice": {
      "cc": 73.93119843479649,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 69.45701357466062,
       "judge": 73.9292364990689,
       "standard": 77.82978199285628
      }
     },
     "sealed|noul": {
      "cc": 24.24849331039893,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 32.00000000000001,
       "hard": 4.854368932038833,
       "judge": 28.947368421052634,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 52.12606956281044,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.31573995848055,
       "hard": 49.879926355330106,
       "judge": 59.15248322255222,
       "standard": 39.98390029032339
      }
     }
    }
   },
   "jevbench_score": 63.1137973454361,
   "key": "spx-cd-omni",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "SurdAI/SPX-CD-Omni@06a786a13f2ce9b8d1b8f541cf030fae007008fb + google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9285714285714286,
    "decisive_rate": 0.6792452830188679,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.47801599949784,
     "cost": 58.27072766098771,
     "intelligence": 48.779308810848946,
     "speed": 89.44891259349035
    },
    "board_release": "v1.6.43",
    "capability": 68.1286624051734,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 32,
    "capability_rank": 53,
    "composite_ci95": {
     "A": [
      49.00994204245296,
      68.5754476769007
     ],
     "B": [
      44.94378680804397,
      65.03574713274071
     ],
     "C": [
      34.03468197392567,
      55.062399131442234
     ]
    },
    "jevbench_score": 63.16406085537614,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 30,
    "rank": 51,
    "ranks": {
     "A": 51,
     "B": 55,
     "C": 59,
     "capability": 90
    },
    "scores": {
     "A": 63.16406085537614,
     "B": 58.9160449775072,
     "C": 43.86393114956675
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2966954869683832,
     "p50_s_raw": 0.0733477434841916,
     "p95_s_adjusted": 0.38264654851518576,
     "p95_s_raw": 0.11632327425759287
    },
    "views": {
     "Balanced 33:33:33": 58.4608318148256,
     "Emphasis on Accuracy 60:20:20": 52.9688184573952,
     "Emphasis on Cost 20:20:60": 57.22245674010682,
     "Emphasis on Speed 20:60:20": 66.8372282634097,
     "Intelligence only": 46.42660407688444
    }
   },
   "provenance": {
    "carried_metadata_from": "r53 PREREG-R53.md, dl_r53.sh and current registry",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 57,
   "ranked": true,
   "ranks": {
    "A": 57,
    "B": 61,
    "C": 68,
    "capability": 105
   },
   "repo": "https://huggingface.co/SurdAI/SPX-CD-Omni",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 63.1137973454361,
    "B": 58.881053876754876,
    "C": 43.829025934330616
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3001056468114257,
    "p50_s_raw": 0.07505282340571284,
    "p95_s_adjusted": 0.4131202141754329,
    "p95_s_raw": 0.13156010708771645
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.85250104999506,
      89.14284725267011
     ],
     "I": [
      43.845760787268546,
      53.75726553555249
     ],
     "capability": [
      64.04521719808471,
      70.65155471508619
     ],
     "composite": {
      "A": [
       48.97252601630862,
       68.51906187054689
      ],
      "B": [
       44.91858495976889,
       64.99516705077025
      ],
      "C": [
       34.00869862243654,
       55.01698761837606
      ]
     },
     "seed": 137
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.47801599949784,
     "capability": 68.1286624051734,
     "intelligence": 48.779308810848946
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7973333333333333,
      "calibration": 90.32414557659362,
      "cc": 74.08182791318927,
      "cc_by_type": {
       "choice": 74.08182791318927
      },
      "cc_tiered": 71.82870406521602,
      "cc_tiered_open": 69.72620969563555,
      "cc_tiered_sealed": 73.93119843479649,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.08182791318927
     },
     "noul": {
      "accuracy": 0.624,
      "calibration": 83.11778989036691,
      "cc": 24.8,
      "cc_by_type": {
       "noul": 24.8
      },
      "cc_tiered": 20.593190754578348,
      "cc_tiered_open": 16.937888198757765,
      "cc_tiered_sealed": 24.24849331039893,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.8
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 88.99211253153302,
      "cc": 52.652284619387025,
      "cc_by_type": {
       "score": 52.652284619387025
      },
      "cc_tiered": 53.91603161275248,
      "cc_tiered_open": 55.70599366269451,
      "cc_tiered_sealed": 52.12606956281044,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.652284619387025
     }
    },
    "raw_on_A": {
     "calibration": 83.85218691107563,
     "capability": 66.88160846988468,
     "intelligence": 49.91103002869373
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 58.40343331211847,
    "Emphasis on Accuracy 60:20:20": 52.94053340136613,
    "Emphasis on Cost 20:20:60": 57.18944785559305,
    "Emphasis on Speed 20:60:20": 66.70232293920729,
    "Intelligence only": 46.42660407688444
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Hangzhi Zhou (@Hangzhi)",
   "axes": {
    "calibration": 61.31489461738089,
    "cost": 49.559714426383636,
    "intelligence": 54.860121638521335,
    "speed": 86.28217783727928
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2516031734669799,
      "ece_score": 49.67936530660402,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 49.67936530660402,
      "score_before_time_limit": 49.67936530660402,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.07027027027027022,
      "brier": 0.14489436428485655,
      "ece": 0.14040055215319622,
      "n": 370,
      "score": 71.91988956936075,
      "score_before_time_limit": 71.91988956936075,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.6811989100817438,
      "interval90_width": 0.0510899182561308,
      "n": 367,
      "nrps": 0.11223140401478648,
      "score": 62.3454289761779,
      "score_before_time_limit": 62.3454289761779,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.3204300082308278
     }
    },
    "score": 61.31489461738089
   },
   "capability": 58.08750812795111,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     52.72222229009074,
     61.19525649627731
    ],
    "B": [
     50.8806792095391,
     60.603262493227525
    ],
    "C": [
     36.61265436811857,
     60.60307970291295
    ]
   },
   "cost": {
    "basis": "ESTIMATE, and the caveat travels with the number rather than sitting in a footnote. The frozen pricing snapshot returns None for google/diffusiongemma-26B-A4B-it (it is unlisted: no exact rate and no floor), and a scan of all 129 registry entries finds NO published row on any DiffusionGemma base at all - the page's openjev-razorback16 is the same model but a different submitter and carries no v1.6 score - so there was no precedent to copy. The reference used is google/gemma-4-26b-a4b-it, USD 0.09 per 1M input and USD 0.30 per 1M output: the identically sized and identically shaped publicly hosted Gemma 4 26B-A4B sibling of the unlisted diffusion checkpoint, a closer sibling than the one run 54 had to use for gemma-4-12B-it. x this row's own measured tokens. THE CAVEAT: a token-priced cost axis charges this row as though it ran one forward pass, when the author's own documentation denoises a 256-position canvas for up to 48 adaptive steps per question. The Cost axis therefore UNDERSTATES this configuration's compute, by construction and not by accident; the measured per-question denoising_steps are kept in the raw so the understatement is quantifiable. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.047275059999999994
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04801123222748815
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Diffusion Jev (DiffusionGemma 26B-A4B on patched SGLang, 48-step diffusion readout, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/Hangzhi/diffusion-jev-sglang@d61f0a9c0c9b6d78631aa250125dc4b20e9ea0a2 serving google/diffusiongemma-26B-A4B-it@f7f5b7f5fa82ffc52addd066915886d497f5517b on SGLang@ef90bf677e4e6b280bf796f1c5f4ef0f05dda27f (their open PR 34061) with their own version-checked readout and context-admission patch. Built from THEIR OWN tested container recipe (deployment/modal_app.py + deployment/gemma-requirements.txt), which their doc names as the recipe to use; their `engine` extra and top-level Dockerfile target the older LLaDA deployment and their doc says plainly not to use them as a DiffusionGemma installer, so they were not used. SGLang is a pure-python install imported through PYTHONPATH, no source compile. Served with their own scripts/launch_diffusiongemma.py: native Gemma4Renoise, a 256-position canvas, up to 48 adaptive denoising steps, seed 42, thinking disabled, 8,192-token context, Triton attention, no CUDA graphs, no prefix cache, T=1 softmax over the options offered - their configuration, unchanged, on one H100 80 GB (Lium) against the A100 80 GB they tested. THE AUTHOR'S OWN STATEMENT, CARRIED UNSOFTENED: these are self-conditioned denoiser scores, NOT independent masked-token likelihoods and NOT calibrated probabilities of correctness; this is not a one-forward-pass or zero-generation method. There are no new trained weights - it is an inference configuration over Google's unchanged checkpoint. A format mismatch returns their own HTTP 503 and is recorded invalid, the same way for everyone. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #30",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 53.207343059355594,
    "I_sealed": 56.51290021768708,
    "base": 54.860121638521335,
    "excess": -5.878944800769732,
    "gap": -3.3055571583314887,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 66.57316141515619,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 60.36489462094999,
       "judge": 56.49307844429796,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 51.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 41.10683877740336,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 42.890763672952836,
       "hard": 43.8594983288807,
       "judge": 51.92844064579445,
       "standard": 18.477154424087306
      }
     },
     "sealed|choice": {
      "cc": 67.58861736481802,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 59.27601809954751,
       "judge": 70.2048417132216,
       "standard": 73.39573839142753
      }
     },
     "sealed|noul": {
      "cc": 53.955192987254144,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 32.038834951456316,
       "judge": 65.78947368421053,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 47.994890300989084,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.90991267549208,
       "hard": 40.176230341477435,
       "judge": 50.19410623577225,
       "standard": 49.875875130586145
      }
     }
    }
   },
   "jevbench_score": 59.26951701831397,
   "key": "diffusion-jev",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0 (model card metadata); linked Gemma 4 model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "google/diffusiongemma-26B-A4B-it@f7f5b7f5fa82ffc52addd066915886d497f5517b",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8546511627906976,
    "decisive_rate": 0.9297297297297298,
    "supported": true,
    "valid": 370
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 61.31489461738089,
     "cost": 49.559714426383636,
     "intelligence": 54.860121638521335,
     "speed": 86.79193858661813
    },
    "board_release": "v1.6.43",
    "capability": 58.08750812795111,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 66,
    "capability_rank": 91,
    "composite_ci95": {
     "A": [
      52.773263175899025,
      61.260192333972526
     ],
     "B": [
      50.91942517195592,
      60.65419970930148
     ],
     "C": [
      36.648099427707656,
      60.66776170099464
     ]
    },
    "jevbench_score": 59.330428224994314,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 34,
    "rank": 58,
    "ranks": {
     "A": 58,
     "B": 56,
     "C": 52,
     "capability": 144
    },
    "scores": {
     "A": 59.330428224994314,
     "B": 58.15811643913557,
     "C": 49.60078097158421
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4113455285318196,
     "p50_s_raw": 0.1306727642659098,
     "p95_s_adjusted": 0.5088612267747521,
     "p95_s_raw": 0.17943061338737604
    },
    "views": {
     "Balanced 33:33:33": 59.033372905673865,
     "Emphasis on Accuracy 60:20:20": 56.86619781829752,
     "Emphasis on Cost 20:20:60": 54.41034813835743,
     "Emphasis on Speed 20:60:20": 67.31868662948479,
     "Intelligence only": 53.898210717803785
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 65,
   "ranked": true,
   "ranks": {
    "A": 65,
    "B": 63,
    "C": 59,
    "capability": 162
   },
   "repo": "https://huggingface.co/google/diffusiongemma-26B-A4B-it",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 59.26951701831397,
    "B": 58.1112837472447,
    "C": 49.5498586453571
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.41225921185687187,
    "p50_s_raw": 0.13112960592843592,
    "p95_s_adjusted": 0.5709681035950779,
    "p95_s_raw": 0.21048405179753896
   },
   "status": {
    "answered_ok": 1474,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/diffusiongemma-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      56.00277250017865,
      64.96308698031199
     ],
     "I": [
      48.25688155312269,
      59.81154987802872
     ],
     "capability": [
      52.99878343969597,
      61.730417849443256
     ],
     "composite": {
      "A": [
       52.72222229009074,
       61.19525649627731
      ],
      "B": [
       50.8806792095391,
       60.603262493227525
      ],
      "C": [
       36.61265436811857,
       60.60307970291295
      ]
     },
     "seed": 50
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 61.31489461738089,
     "capability": 58.08750812795111,
     "intelligence": 54.860121638521335
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7533333333333333,
      "calibration": 49.67936530660402,
      "cc": 68.45485634171062,
      "cc_by_type": {
       "choice": 68.45485634171062
      },
      "cc_tiered": 67.0808893899871,
      "cc_tiered_open": 66.57316141515619,
      "cc_tiered_sealed": 67.58861736481802,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 68.45485634171062
     },
     "noul": {
      "accuracy": 0.784,
      "calibration": 71.91988956936075,
      "cc": 56.800000000000004,
      "cc_by_type": {
       "noul": 56.800000000000004
      },
      "cc_tiered": 52.9486109863807,
      "cc_tiered_open": 51.94202898550725,
      "cc_tiered_sealed": 53.955192987254144,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.800000000000004
     },
     "score": {
      "accuracy": 0.6106666666666667,
      "calibration": 62.3454289761779,
      "cc": 47.39565350609251,
      "cc_by_type": {
       "score": 47.39565350609251
      },
      "cc_tiered": 44.550864539196226,
      "cc_tiered_open": 41.10683877740336,
      "cc_tiered_sealed": 47.994890300989084,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.39565350609251
     }
    },
    "raw_on_A": {
     "calibration": 57.69524290701848,
     "capability": 57.542251645635915,
     "intelligence": 57.38926038425334
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.021333333333333315,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 58.95299613597719,
    "Emphasis on Accuracy 60:20:20": 56.821421889044025,
    "Emphasis on Cost 20:20:60": 54.36935473248989,
    "Emphasis on Speed 20:60:20": 67.13081675757532,
    "Intelligence only": 53.898210717803785
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Elon Salfati / salfatigroup",
   "axes": {
    "calibration": 90.18677086647014,
    "cost": 45.963638345890566,
    "intelligence": 61.72139577013068,
    "speed": 86.74281071257914
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.043257328990227974,
      "ece_score": 91.34853420195441,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 91.34853420195441,
      "score_before_time_limit": 91.34853420195441,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.22102425876010778,
      "brier": 0.06278221024258757,
      "ece": 0.05196765498652292,
      "n": 371,
      "score": 89.60646900269542,
      "score_before_time_limit": 89.60646900269542,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.3859530261969287,
      "n": 369,
      "nrps": 0.06312416440831078,
      "score": 89.60530939476061,
      "score_before_time_limit": 89.60530939476061,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07238482384823842
     }
    },
    "score": 90.18677086647014
   },
   "capability": 75.9540833183004,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     54.20725086179235,
     56.57484017254628
    ],
    "B": [
     52.75897286617128,
     56.387398794479765
    ],
    "C": [
     47.78726660265739,
     56.57484017254628
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference, not charged). Unmodified pricing_v15.reference resolves google/gemma-4-31B-it to google/gemma-4-31b-it at USD0.09/M input and USD0.34/M output in frozen snapshot. Measured output tokens are 0 by construction (decision_contract.py and release.json); one forward-pass option readout. Common v1.6.1 cost input range applied unchanged.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.062301899999999986
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06327207176709546
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "René-1 31B FP8 (salfatigroup, Gemma 4 31B full fine-tune, one-pass option readout)",
   "endpoint_condition": "Candidate re-measured on 2026-10-07, current v1.6.0/v1.6.1 1500-item pool (sha256901983ae...). Immutable salfatigroup/rene-1-31b-fp8@bd634489957f8da63ccce858ee33fd6c2929d286, Apache-2.0. Same unmodified reviewed contributor source and run-39 transport shim. Author noul/yesno type mapping only; two-decimal probability rounding retained. One evaluator-owned RTX PRO6000 Blackwell 96GB pod, loopback-only network namespace, offline credential-free inference; no external hosted API, no retry or batching. Unmodified run_v16/typesafe adapter, serial one request at a time, selfhosted x2+0.15s latency adjustment. Missing ATTRIBUTIONS.md and NOTICE disclosed: training-data overlap cannot be checked. Florian GO in DECISIONS.md (go gor it, Apache2.0 confirmed) resolves prior attribution hold; publication remains release lane responsibility. COMPLETE: 1500/1500 distinct input IDs, 1477 HTTP200 native answers,23 HTTP422 refusals under the author's own32768-token state+question /65536-token request budget; every refusal counted wrong. Smoke passed all8 checks. All42 release-manifest files and6 prior source files hash-verified. Raw sha256 a28a8f0068f38b548f064a6e678bf63b13304f9add0a67b448dcc8003c9f38f1. Stack torch2.14.0, transformers5.17.0, compressed-tensors0.19.0, accelerate1.15.0. Provider pod91916684-9980-419c-98b7-c7a714a3912c removed and verified absent; approxUSD0.17. Fresh latest feed places this candidate inside both official caps; material open-board top-five change requires release-lane preview disposition before publication.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000 Blackwell",
   "intelligence": {
    "I_open": 62.91896990676746,
    "I_sealed": 60.523821633493895,
    "base": 61.72139577013068,
    "excess": -0.17823936916468108,
    "gap": 2.3951482732735627,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 78.65680231931067,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396666,
       "judge": 84.17930125247197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 53.325051759834366,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 56.77505564115734,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.5151515151515,
       "hard": 65.03703703703704,
       "judge": 54.15254237288135,
       "standard": 41.8148148148148
      }
     },
     "sealed|choice": {
      "cc": 78.92231555231959,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 71.49321266968325,
       "judge": 86.96461824953444,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 47.36482975523042,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80.0,
       "hard": 32.038834951456316,
       "judge": 50.0,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 55.28431959293169,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.97163120567376,
       "hard": 50.97549019607843,
       "judge": 64.15568862275447,
       "standard": 43.75126903553299
      }
     }
    }
   },
   "jevbench_score": 55.803088091905906,
   "key": "rene-1-31b-fp8",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "salfatigroup/rene-1-31b-fp8@bd634489957f8da63ccce858ee33fd6c2929d286",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9826989619377162,
    "decisive_rate": 0.7789757412398922,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.18677086647014,
     "cost": 45.963638345890566,
     "intelligence": 61.72139577013068,
     "speed": 87.11720932683018
    },
    "board_release": "v1.6.43",
    "capability": 75.9540833183004,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 12,
    "capability_rank": 22,
    "composite_ci95": {
     "A": [
      54.25035390810508,
      56.621792265328125
     ],
     "B": [
      52.791631605081,
      56.42470571115517
     ],
     "C": [
      47.82528165394186,
      56.621792265328125
     ]
    },
    "jevbench_score": 55.84876743320955,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 36,
    "rank": 60,
    "ranks": {
     "A": 60,
     "B": 59,
     "C": 47,
     "capability": 40
    },
    "scores": {
     "A": 55.84876743320955,
     "B": 55.06950723833363,
     "C": 55.84876743320955
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3694145163521171,
     "p50_s_raw": 0.10970725817605853,
     "p95_s_adjusted": 0.5257328446954489,
     "p95_s_raw": 0.18786642234772444
    },
    "views": {
     "Balanced 33:33:33": 51.281219378577866,
     "Emphasis on Accuracy 60:20:20": 51.62853971578034,
     "Emphasis on Cost 20:20:60": 45.45809395907335,
     "Emphasis on Speed 20:60:20": 58.36507393476142,
     "Intelligence only": 52.15843207508249
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 66,
   "ranked": true,
   "ranks": {
    "A": 66,
    "B": 66,
    "C": 51,
    "capability": 52
   },
   "repo": "https://huggingface.co/salfatigroup/rene-1-31b-fp8",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 55.803088091905906,
    "B": 55.03397029830104,
    "C": 55.803088091905906
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.37661420051008465,
    "p50_s_raw": 0.11330710025504231,
    "p95_s_adjusted": 0.5621111983433366,
    "p95_s_raw": 0.20605559917166827
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.96895555807659,
      90.68721224415906
     ],
     "I": [
      56.323580074482344,
      65.99929373375565
     ],
     "capability": [
      72.10629528396096,
      77.53024725815187
     ],
     "composite": {
      "A": [
       54.20725086179235,
       56.57484017254628
      ],
      "B": [
       52.75897286617128,
       56.387398794479765
      ],
      "C": [
       47.78726660265739,
       56.57484017254628
      ]
     },
     "seed": 128
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.18677086647014,
     "capability": 75.9540833183004,
     "intelligence": 61.72139577013068
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8346666666666667,
      "calibration": 91.34853420195441,
      "cc": 78.85622803444389,
      "cc_by_type": {
       "choice": 78.85622803444389
      },
      "cc_tiered": 78.78955893581514,
      "cc_tiered_open": 78.65680231931067,
      "cc_tiered_sealed": 78.92231555231959,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 78.85622803444389
     },
     "noul": {
      "accuracy": 0.7573333333333333,
      "calibration": 89.60646900269542,
      "cc": 51.46666666666666,
      "cc_by_type": {
       "noul": 51.46666666666666
      },
      "cc_tiered": 50.34494075753239,
      "cc_tiered_open": 53.325051759834366,
      "cc_tiered_sealed": 47.36482975523042,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 51.46666666666666
     },
     "score": {
      "accuracy": 0.6746666666666666,
      "calibration": 89.60530939476061,
      "cc": 55.4684210526316,
      "cc_by_type": {
       "score": 55.4684210526316
      },
      "cc_tiered": 56.029687617044516,
      "cc_tiered_open": 56.77505564115734,
      "cc_tiered_sealed": 55.28431959293169,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.4684210526316
     }
    },
    "raw_on_A": {
     "calibration": 86.48256493925484,
     "capability": 76.2896760781785,
     "intelligence": 66.09678721710216
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 51.22987789575082,
    "Emphasis on Accuracy 60:20:20": 51.5973037568403,
    "Emphasis on Cost 20:20:60": 45.43387647578042,
    "Emphasis on Speed 20:60:20": 58.245489518415575,
    "Intelligence only": 52.15843207508249
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "EmpirioLabs (@Adam-Dalloul)",
   "axes": {
    "calibration": 88.94185474430773,
    "cost": 57.86508463809988,
    "intelligence": 45.23340779864823,
    "speed": 88.75158098066062
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06972206212499996,
      "ece_score": 86.055587575,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 85.7113652247,
      "score_before_time_limit": 86.055587575,
      "time_limit_failed_share": 0.004
     },
     "noul": {
      "abstention_rate": 0.31733333333333336,
      "brier": 0.11392156247660881,
      "ece": 0.04671065970666656,
      "n": 375,
      "score": 90.41611374384358,
      "score_before_time_limit": 90.6578680586667,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.9706666666666667,
      "interval90_width": 0.4937777777777778,
      "n": 375,
      "nrps": 0.09021695431167416,
      "score": 90.69808526437963,
      "score_before_time_limit": 90.94059351374963,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.045485587706666636
     }
    },
    "score": 88.94185474430773
   },
   "capability": 67.08763127147799,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     38.628270158039165,
     64.8203588915082
    ],
    "B": [
     34.84566256928663,
     60.62209766423049
    ],
    "C": [
     26.825187609749417,
     45.014138119102924
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights): the exact declared base Qwen/Qwen3.5-4B resolves in the frozen pricing snapshot through DEEPINFRA_BASE_REFERENCES to USD 0.03 per 1M input and USD 0.15 per 1M output, read before the pod and asserted in make_registry_r55.py. The same rate run 54 gave messier-one, the other self-hosted Qwen3.5-4B derivative. DELIBERATELY NOT the vendor's own cheaper public tariff: Aplomb 1 is also sold on EmpirioLabs' hosted API at USD 0.02 per 1M input with output free (GitHub #195), which would LOWER this row's cost - but all 106 self-hosted rows in the registry carry cost_kind `estimate` and not one is priced at a vendor tariff, and creating that precedent for one row is the release lane's call, not this job's. The estimate can therefore only OVERSTATE this row, and it is reversible by arithmetic because the raw keeps the tokens. x this row's own measured tokens. INPUT TOKENS ARE EXACT, NOT ESTIMATED, and they include BOTH passes of the author's own --debias flag on every multi-option choice, because that is what the configuration consumes. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the author's answer_logits runs the model with logits_to_keep=1, use_cache=False and reads label logits, and nothing is generated anywhere in their code path. Counts carried in the scorer's own cost_estimate field, never written into usage. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.08184117999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025380324983073795
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 5,
    "n_over_limit_answered_ok": 5,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Aplomb 1 (5.3B decision model on Qwen3.5-4B, trained readout head, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights empiriolabsai/aplomb-1@f8e2dc81f0505db21a3b7a933ea479658e4ebb8f, bf16, 5.3 B, base Qwen/Qwen3.5-4B plus the shipped 1.3 GB audio encoder and the trained DecisionHead; click-through gate accepted under the EmpirioLabs Model License 1.0 section 2.1, which grants evaluation and the publication of benchmark results without asking. Served behind the author's own TypeSafe wire format by a TRANSPORT SHIM ONLY: run_aplomb.main()'s loading block verbatim, every request handed to THEIR run_aplomb.decide() with THEIR edm_config.json temperatures and abstain_bias and debias=True - the configuration the author names as the reference in #195 and in the model card. Their own aplomb/schema.py documents the request schema as wire-compatible with TypeSafe /v1/systemone and their aplomb/readout.py already emits the answer dicts the unchanged typesafe adapter parses, so no answer field is rewritten and no contributor file is edited. STACK: the author's own documented install line, with transformers pinned to 5.17.0 - the version their own audio sidecar was saved with; their loader compares the saved config to the live one and 5.19.0 fails it on the transformers_version string alone. SPEED CAVEAT: their documented install line does not include causal_conv1d, so transformers falls back to a reference PyTorch implementation it itself calls much slower; the Speed axis is measured on the stack the author documents. One H100 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #195",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 48.14750387886506,
    "I_sealed": 42.31931171843139,
    "base": 45.23340779864823,
    "excess": 3.254804517995422,
    "gap": 5.828192160433666,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.2456274917742,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 67.9144385026738,
       "judge": 80.22412656558997,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 26.777087646652866,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 47.419796498168125,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.31956009090909,
       "hard": 46.33070064814815,
       "judge": 45.652502593220326,
       "standard": 47.299047259259254
      }
     },
     "sealed|choice": {
      "cc": 61.23201765527931,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 56.730769230769226,
       "judge": 62.756052141527014,
       "standard": 61.20211848749847
      }
     },
     "sealed|noul": {
      "cc": 22.806658654019163,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -4.854368932038833,
       "judge": 36.8421052631579,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 42.919258845995714,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.11917736170213,
       "hard": 38.489070343137286,
       "judge": 48.63257153293411,
       "standard": 33.60970756345175
      }
     }
    }
   },
   "jevbench_score": 52.8870189100091,
   "key": "aplomb-1",
   "last_measured_on": "2026-10-06",
   "licence": "EmpirioLabs Model License 1.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "empiriolabsai/aplomb-1@f8e2dc81f0505db21a3b7a933ea479658e4ebb8f",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9333333333333333,
    "decisive_rate": 0.6818181818181818,
    "supported": true,
    "valid": 374
   },
   "open": "gated weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.21801638247211,
     "cost": 57.86508463809988,
     "intelligence": 45.81854920739271,
     "speed": 88.88310989631262
    },
    "board_release": "v1.6.43",
    "capability": 67.51828279493242,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 33,
    "capability_rank": 54,
    "composite_ci95": {
     "A": [
      40.003677562543594,
      66.59206455090654
     ],
     "B": [
      36.14823252368513,
      62.39027598829859
     ],
     "C": [
      27.780331640655273,
      46.3455307690425
     ]
    },
    "jevbench_score": 54.55838408944504,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 37,
    "rank": 61,
    "ranks": {
     "A": 61,
     "B": 64,
     "C": 69,
     "capability": 93
    },
    "scores": {
     "A": 54.55838408944504,
     "B": 50.34915295427923,
     "C": 37.88776672878128
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.33727095890790226,
     "p50_s_raw": 0.09363547945395112,
     "p95_s_adjusted": 0.3834511662367731,
     "p95_s_raw": 0.11672558311838657
    },
    "views": {
     "Balanced 33:33:33": 50.026426143216206,
     "Emphasis on Accuracy 60:20:20": 44.66300743458741,
     "Emphasis on Cost 20:20:60": 49.44235580546631,
     "Emphasis on Speed 20:60:20": 57.6274614302068,
     "Intelligence only": 38.475475184084495
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 70,
   "ranked": true,
   "ranks": {
    "A": 70,
    "B": 72,
    "C": 80,
    "capability": 108
   },
   "repo": "https://huggingface.co/empiriolabsai/aplomb-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 52.8870189100091,
    "B": 48.7114560128917,
    "C": 36.72709646528409
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3351494163274765,
    "p50_s_raw": 0.09257470816373825,
    "p95_s_adjusted": 0.3977438145782798,
    "p95_s_raw": 0.12387190728913991
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.59491561004869,
      89.59945632360366
     ],
     "I": [
      39.712930281345386,
      49.434470191151824
     ],
     "capability": [
      62.49978696897133,
      68.99973905641266
     ],
     "composite": {
      "A": [
       38.628270158039165,
       64.8203588915082
      ],
      "B": [
       34.84566256928663,
       60.62209766423049
      ],
      "C": [
       26.825187609749417,
       45.014138119102924
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.94185474430773,
     "capability": 67.08763127147799,
     "intelligence": 45.23340779864823
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7173333333333334,
      "calibration": 85.7113652247,
      "cc": 63.85097051050084,
      "cc_by_type": {
       "choice": 63.85097051050084
      },
      "cc_tiered": 65.73882257352676,
      "cc_tiered_open": 70.2456274917742,
      "cc_tiered_sealed": 61.23201765527931,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.85097051050084
     },
     "noul": {
      "accuracy": 0.6346666666666667,
      "calibration": 90.41611374384358,
      "cc": 26.933333333333344,
      "cc_by_type": {
       "noul": 26.933333333333344
      },
      "cc_tiered": 24.791873150336016,
      "cc_tiered_open": 26.777087646652866,
      "cc_tiered_sealed": 22.806658654019163,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 26.933333333333344
     },
     "score": {
      "accuracy": 0.608,
      "calibration": 90.69808526437963,
      "cc": 43.92186809298247,
      "cc_by_type": {
       "score": 43.92186809298247
      },
      "cc_tiered": 45.16952767208192,
      "cc_tiered_open": 47.419796498168125,
      "cc_tiered_sealed": 42.919258845995714,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.92186809298247
     }
    },
    "raw_on_A": {
     "calibration": 89.64105266703848,
     "capability": 70.52705142972752,
     "intelligence": 51.413050192416556
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0040000000000000036,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 48.469033821173,
    "Emphasis on Accuracy 60:20:20": 43.13324979063626,
    "Emphasis on Cost 20:20:60": 48.01850560949906,
    "Emphasis on Speed 20:60:20": 55.90992002519982,
    "Intelligence only": 37.02012791388867
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 77.04569964036838,
    "cost": 46.094951684283686,
    "intelligence": 56.54011796945148,
    "speed": 89.00367576375322
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1589499873699225,
      "ece_score": 68.2100025260155,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 68.2100025260155,
      "score_before_time_limit": 68.2100025260155,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.13477088948787064,
      "brier": 0.12391951730825534,
      "ece": 0.10295952013212975,
      "n": 371,
      "score": 79.40809597357405,
      "score_before_time_limit": 79.40809597357405,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8699186991869918,
      "interval90_width": 0.20505871725383928,
      "n": 369,
      "nrps": 0.07833002311655612,
      "score": 83.51900042151561,
      "score_before_time_limit": 83.51900042151561,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1256449842265658
     }
    },
    "score": 77.04569964036838
   },
   "capability": 66.79290880490993,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     51.067572471880325,
     54.799559247976
    ],
    "B": [
     49.06715119828214,
     54.19776091868999
    ],
    "C": [
     35.724842970712416,
     54.79865331021095
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter qwen/qwen3.5-9b, the reference of the published nimble-9b row (same base), USD 0.1 per 1M input and USD 0.15 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.06167713333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06263757616790792
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble 9B v3 (Qwen3.5-9B LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.07198447174714,
    "I_sealed": 56.008251467155816,
    "base": 56.54011796945148,
    "excess": -1.5096546378469164,
    "gap": 1.0637330045913274,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.11673425629002,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 60.36489462094999,
       "judge": 76.26895187870797,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 48.59075224292616,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 39.99999999999999,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 50.508466916025256,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.68883649626082,
       "hard": 50.44357111216904,
       "judge": 56.6073538275629,
       "standard": 29.899743366313437
      }
     },
     "sealed|choice": {
      "cc": 63.08410687973317,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 58.76696832579185,
       "judge": 60.89385474860336,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 51.038414647311576,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 28.155339805825253,
       "judge": 65.78947368421053,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 53.90223287442272,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 75.56838262623131,
       "hard": 43.523681374236745,
       "judge": 58.275920598763435,
       "standard": 57.26572941237931
      }
     }
    }
   },
   "jevbench_score": 53.456301822552156,
   "key": "bespoke-nimble-9b-v3",
   "last_measured_on": "2026-10-06",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "bespokelabs/Bespoke-Nimble-9B-v3@8e927b9b4afdbb14479fac10a7364d1a695be208",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8940809968847352,
    "decisive_rate": 0.8652291105121294,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.04569964036838,
     "cost": 46.094951684283686,
     "intelligence": 56.54011796945148,
     "speed": 89.19502471208428
    },
    "board_release": "v1.6.43",
    "capability": 66.79290880490993,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 34,
    "capability_rank": 55,
    "composite_ci95": {
     "A": [
      51.086235675343055,
      54.820858958312876
     ],
     "B": [
      49.08081410151091,
      54.214427134732105
     ],
     "C": [
      35.737762455481224,
      54.819952316161036
     ]
    },
    "jevbench_score": 53.47656993239491,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 39,
    "rank": 63,
    "ranks": {
     "A": 63,
     "B": 60,
     "C": 56,
     "capability": 94
    },
    "scores": {
     "A": 53.47656993239491,
     "B": 52.29615241766553,
     "C": 47.486970389641556
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.33777692201547327,
     "p50_s_raw": 0.09388846100773662,
     "p95_s_adjusted": 0.3563424444932025,
     "p95_s_raw": 0.10317122224660125
    },
    "views": {
     "Balanced 33:33:33": 50.39687437311006,
     "Emphasis on Accuracy 60:20:20": 49.43254820883513,
     "Emphasis on Cost 20:20:60": 45.21647399422632,
     "Emphasis on Speed 20:60:20": 58.20016580649009,
     "Intelligence only": 48.05332347466076
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 69,
   "ranked": true,
   "ranks": {
    "A": 69,
    "B": 67,
    "C": 63,
    "capability": 109
   },
   "repo": "https://huggingface.co/bespokelabs/Bespoke-Nimble-9B-v3",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 53.456301822552156,
    "B": 52.28064457258714,
    "C": 47.46897239288946
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3390120119787753,
    "p50_s_raw": 0.09450600598938763,
    "p95_s_adjusted": 0.37103710017108815,
    "p95_s_raw": 0.11051855008554409
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.11138007423693,
      79.85549338743986
     ],
     "I": [
      49.993162280566764,
      61.190765007174605
     ],
     "capability": [
      61.91008408542518,
      69.96614101398373
     ],
     "composite": {
      "A": [
       51.067572471880325,
       54.799559247976
      ],
      "B": [
       49.06715119828214,
       54.19776091868999
      ],
      "C": [
       35.724842970712416,
       54.79865331021095
      ]
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.04569964036838,
     "capability": 66.79290880490993,
     "intelligence": 56.54011796945148
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 68.2100025260155,
      "cc": 65.89714199103851,
      "cc_by_type": {
       "choice": 65.89714199103851
      },
      "cc_tiered": 67.6004205680116,
      "cc_tiered_open": 72.11673425629002,
      "cc_tiered_sealed": 63.08410687973317,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.89714199103851
     },
     "noul": {
      "accuracy": 0.7653333333333333,
      "calibration": 79.40809597357405,
      "cc": 53.06666666666666,
      "cc_by_type": {
       "noul": 53.06666666666666
      },
      "cc_tiered": 49.81458344511887,
      "cc_tiered_open": 48.59075224292616,
      "cc_tiered_sealed": 51.038414647311576,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.06666666666666
     },
     "score": {
      "accuracy": 0.6586666666666666,
      "calibration": 83.51900042151561,
      "cc": 53.90613650563776,
      "cc_by_type": {
       "score": 53.90613650563776
      },
      "cc_tiered": 52.20534989522399,
      "cc_tiered_open": 50.508466916025256,
      "cc_tiered_sealed": 53.90223287442272,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.90613650563776
     }
    },
    "raw_on_A": {
     "calibration": 79.9777663718046,
     "capability": 70.03747268625314,
     "intelligence": 60.09717900070167
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 50.37287555368566,
    "Emphasis on Accuracy 60:20:20": 49.41869198062401,
    "Emphasis on Cost 20:20:60": 45.2048802744577,
    "Emphasis on Speed 20:60:20": 58.14258462219303,
    "Intelligence only": 48.05332347466076
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "APUS AI-LAB (@cryptoSUN2049 / findoo2049@gmail.com)",
   "axes": {
    "calibration": 66.61054590286017,
    "cost": 48.738123569705756,
    "intelligence": 47.20514429531085,
    "speed": 82.0651268880946
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21593843550946115,
      "ece_score": 56.81231289810778,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 56.81231289810778,
      "score_before_time_limit": 56.81231289810778,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.1185983827493261,
      "brier": 0.1586174454486672,
      "ece": 0.14320810756172592,
      "n": 371,
      "score": 71.35837848765482,
      "score_before_time_limit": 71.35837848765482,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8401084010840109,
      "interval90_width": 0.29200542005420044,
      "n": 369,
      "nrps": 0.12208457963241476,
      "score": 71.66094632281795,
      "score_before_time_limit": 71.66094632281795,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.22234824695561312
     }
    },
    "score": 66.61054590286017
   },
   "capability": 56.907845099085506,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     32.99548219792953,
     56.557757569850644
    ],
    "B": [
     30.74220833343163,
     54.676966440426924
    ],
    "C": [
     22.913529304117727,
     40.18283169934896
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights): the exact declared base Qwen/Qwen3.5-9B resolves in the frozen pricing snapshot through BASE_REFERENCES to qwen/qwen3.5-9b = USD 0.10 per 1M input and USD 0.15 per 1M output, read before the pod and asserted in make_registry_r56.py. Nothing provisional. x this row's own measured tokens. BECAUSE OUTPUT TOKENS ARE 0 BY CONSTRUCTION this axis rests on the input rate alone. Estimated, not charged.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0503522
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05113628977657414
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "APUS-OpenJev-v1-9B (merged bf16 checkpoint-3000 on Qwen3.5-9B, their own native runtime, full depth, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights apus-ailab/APUS-OpenJev-v1-9B@82c9c56cfa9de8d36704ed91948d4726ef111635, Apache-2.0, 17.55 GiB bf16 merged weights, checkpoint-3000, declared base Qwen/Qwen3.5-9B, full depth 32 layers. STACK: the authors' own requirements.txt, unmodified - torch==2.8.0 and transformers==5.16.1. The transformers pin is THEIRS, not a workaround: OpenJet.__init__ raises 'Layer execution is audited for transformers==5.16.1' on any other version. Served behind a TRANSPORT SHIM ONLY (r56/serve/apus_systemone_r56.py): loaded with THEIR OpenJet.from_pretrained(device=cuda:0, dtype=bfloat16) and every request handed to THEIR decide(), returning THEIR format_response() output. No contributor file edited. effort='high', the full-depth path, their decide() default and the configuration their own headline Frozen80 number is quoted on; effort='low' is not measured and no routing between the two is invented. One H100 PCIe 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. MAPPING (run 56's, not the authors'; pre-registered in PREREG-R56 s2.2 before any item was sent and repeated here because it is part of the number): their runtime accepts primitive in {choice, noul, score_level} with 2-16 candidates labelled A-P. choice -> their 'choice' primitive, candidates = the item's own labels as id and its own criteria[label] as description, in labels order; their format_response returns probabilities keyed by candidate id, i.e. by the JevBench label already, so nothing is rewritten. All 750 choice items carry 3-9 options, inside their 2-16 window. noul -> their 'noul' primitive; their contract REQUIRES the canonical yes/no criteria there, so the item's own true/false description strings are NOT delivered - the proposition itself travels verbatim in instructions, which is what their noul is built to judge. score -> their 'choice' primitive over the ordinal levels, because their own RUNTIME.md rules score_level out as an ordinal Score API ('score_level is the judgement of a single proposition, it cannot be treated as a complete ordinal Score API'); the 4-5 ordered levels become candidates with id = the level index and description = the item's own level text, which carries the order. Recording all 375 score items as misses instead would understate a system whose published interface answers them. CALIBRATION IS MEASURED AS THE AUTHORS SHIP IT AND THEY DISCLOSED IT IN ADVANCE: their response carries calibrated: False and both their card and RUNTIME.md state the probabilities are an uncalibrated relative softmax over the offered candidates. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, NOT BY ESTIMATE: decide() runs one forward pass with logits_to_keep=1 and reads candidate logits; nothing is generated anywhere in that path. Input tokens are their own prompt_tokens field. THE 23 LONG ITEMS: their compile() raises ValueError('input exceeds runtime limit; no truncation permitted') above 8,192 tokens and their RUNTIME.md states it in advance. Recorded invalid. This is the first row in the series whose refusal on those items is guaranteed in the author's own code with an explicit no-truncation promise. add-requests run 56, GitHub #106, benchmarkheaven.com/submit ref 2033065A",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100 PCIe",
   "intelligence": {
    "I_open": 52.06268273300428,
    "I_sealed": 42.347605857617424,
    "base": 47.20514429531085,
    "excess": 7.141689232948613,
    "gap": 9.715076875386856,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.13339106992548,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 48.58272907053395,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 59.49758454106281,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 60.00000000000001,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.55707258802457,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.48792439337757,
       "hard": 38.515870391630514,
       "judge": 34.91730621356864,
       "standard": 23.633700639820077
      }
     },
     "sealed|choice": {
      "cc": 56.01292563590547,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 44.00452488687781,
       "judge": 59.96275605214154,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 39.873501407011304,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 20.388349514563096,
       "judge": 39.47368421052631,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 31.156390529935493,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.76837620384379,
       "hard": 19.973469764155748,
       "judge": 28.021772929225364,
       "standard": 48.418165625606015
      }
     }
    }
   },
   "jevbench_score": 49.167351539114094,
   "key": "apus-openjev-v1-9b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "apus-ailab/APUS-OpenJev-v1-9B@82c9c56cfa9de8d36704ed91948d4726ef111635",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8470948012232415,
    "decisive_rate": 0.8814016172506739,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 66.61054590286017,
     "cost": 48.738123569705756,
     "intelligence": 47.20514429531085,
     "speed": 82.18245466956745
    },
    "board_release": "v1.6.43",
    "capability": 56.907845099085506,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 74,
    "capability_rank": 100,
    "composite_ci95": {
     "A": [
      33.00331563170442,
      56.572404110680004
     ],
     "B": [
      30.747622136678878,
      54.687915864792735
     ],
     "C": [
      22.918969188683615,
      40.193299875459424
     ]
    },
    "jevbench_score": 49.17976892452783,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 41,
    "rank": 65,
    "ranks": {
     "A": 65,
     "B": 66,
     "C": 71,
     "capability": 154
    },
    "scores": {
     "A": 49.17976892452783,
     "B": 47.01556694203153,
     "C": 34.15261730869988
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.705569653166458,
     "p50_s_raw": 0.277784826583229,
     "p95_s_adjusted": 0.8574615204473958,
     "p95_s_raw": 0.35373076022369787
    },
    "views": {
     "Balanced 33:33:33": 47.16399064408527,
     "Emphasis on Accuracy 60:20:20": 44.00058844315432,
     "Emphasis on Cost 20:20:60": 44.61842645447928,
     "Emphasis on Speed 20:60:20": 54.145911082892475,
     "Intelligence only": 39.978415802185516
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 73,
   "ranked": true,
   "ranks": {
    "A": 73,
    "B": 74,
    "C": 82,
    "capability": 171
   },
   "repo": "https://huggingface.co/apus-ailab/APUS-OpenJev-v1-9B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 49.167351539114094,
    "B": 47.00648755922215,
    "C": 34.14399412438478
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7130538010736928,
    "p50_s_raw": 0.28152690053684637,
    "p95_s_adjusted": 0.8716959172161296,
    "p95_s_raw": 0.3608479586080648
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.07882152511099,
      70.58159568750249
     ],
     "I": [
      39.88317778412442,
      50.523530007279774
     ],
     "capability": [
      51.79741923709466,
      59.84489784641873
     ],
     "composite": {
      "A": [
       32.99548219792953,
       56.557757569850644
      ],
      "B": [
       30.74220833343163,
       54.676966440426924
      ],
      "C": [
       22.913529304117727,
       40.18283169934896
      ]
     },
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.61054590286017,
     "capability": 56.907845099085506,
     "intelligence": 47.20514429531085
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6666666666666666,
      "calibration": 56.81231289810778,
      "cc": 57.37142748879814,
      "cc_by_type": {
       "choice": 57.37142748879814
      },
      "cc_tiered": 58.57315835291547,
      "cc_tiered_open": 61.13339106992548,
      "cc_tiered_sealed": 56.01292563590547,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.37142748879814
     },
     "noul": {
      "accuracy": 0.7386666666666667,
      "calibration": 71.35837848765482,
      "cc": 47.73333333333334,
      "cc_by_type": {
       "noul": 47.73333333333334
      },
      "cc_tiered": 49.685542974037055,
      "cc_tiered_open": 59.49758454106281,
      "cc_tiered_sealed": 39.873501407011304,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 47.73333333333334
     },
     "score": {
      "accuracy": 0.5306666666666666,
      "calibration": 71.66094632281795,
      "cc": 33.347875357101564,
      "cc_by_type": {
       "score": 33.347875357101564
      },
      "cc_tiered": 33.35673155898003,
      "cc_tiered_open": 35.55707258802457,
      "cc_tiered_sealed": 31.156390529935493,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.347875357101564
     }
    },
    "raw_on_A": {
     "calibration": 70.50531100654376,
     "capability": 61.66232173095365,
     "intelligence": 52.819332455363536
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 47.14876462179523,
    "Emphasis on Accuracy 60:20:20": 43.99263609619584,
    "Emphasis on Cost 20:20:60": 44.610249233224714,
    "Emphasis on Speed 20:60:20": 54.109801729455015,
    "Intelligence only": 39.978415802185516
   }
  },
  {
   "adapter_id": "images/plumb-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 87.80311208802716,
    "cost": 63.07359395281681,
    "intelligence": 42.83684414340664,
    "speed": 93.68902806065091
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.061775972135365,
      "ece_score": 87.644805572927,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 87.41108609139918,
      "score_before_time_limit": 87.644805572927,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0.29333333333333333,
      "brier": 0.12494153723477718,
      "ece": 0.06617846187638739,
      "n": 375,
      "score": 86.76430762472252,
      "score_before_time_limit": 86.76430762472252,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9573333333333334,
      "interval90_width": 0.49066666666666664,
      "n": 375,
      "nrps": 0.09744572815253887,
      "score": 89.2339425479598,
      "score_before_time_limit": 89.2339425479598,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.058937710444132485
     }
    },
    "score": 87.80311208802716
   },
   "capability": 65.3199781157169,
   "class": "unclassified",
   "composite_ci95": {
    "A": [
     33.81211971290175,
     61.75277872389874
    ],
    "B": [
     29.947742194356437,
     57.08354879011454
    ],
    "C": [
     23.480638689515107,
     42.88387411381857
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01701689039408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Plumb-4B (crh225, JevK5 v0.2 + LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 44.9020239650968,
    "I_sealed": 40.77166432171647,
    "base": 42.83684414340664,
    "excess": 1.5569720009420869,
    "gap": 4.130359643380331,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 62.95060504058115,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 52.815350739226176,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 36.11249137336094,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.64297548134833,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.38519253276966,
       "hard": 30.154558838586563,
       "judge": 43.95614933948649,
       "standard": 21.77893945395395
      }
     },
     "sealed|choice": {
      "cc": 58.16335833730948,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 57.239819004524875,
       "judge": 57.16945996275605,
       "standard": 57.876585786426915
      }
     },
     "sealed|noul": {
      "cc": 23.383111546129097,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": 4.854368932038833,
       "judge": 36.8421052631579,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 40.76852308171083,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.90236427761113,
       "hard": 40.87931496804801,
       "judge": 36.35703355260749,
       "standard": 34.597253004741354
      }
     }
    }
   },
   "jevbench_score": 47.92548318441584,
   "key": "plumb-4b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9132075471698113,
    "decisive_rate": 0.7066666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.88101858186977,
     "cost": 63.07359395281681,
     "intelligence": 42.9626698747687,
     "speed": 93.83356539601934
    },
    "board_release": "v1.6.43",
    "capability": 65.42184422831923,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 37,
    "capability_rank": 58,
    "composite_ci95": {
     "A": [
      34.218232721078074,
      62.094303811225494
     ],
     "B": [
      30.257952435880583,
      57.342398440511
     ],
     "C": [
      23.762661611859777,
      43.121044313351035
     ]
    },
    "jevbench_score": 48.28224126395837,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 43,
    "rank": 67,
    "ranks": {
     "A": 67,
     "B": 69,
     "C": 73,
     "capability": 100
    },
    "scores": {
     "A": 48.28224126395837,
     "B": 43.7169924526014,
     "C": 33.529334211082194
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18999176411889493,
     "p50_s_raw": 0.019995882059447467,
     "p95_s_adjusted": 0.21772519345395266,
     "p95_s_raw": 0.033862596726976335
    },
    "views": {
     "Balanced 33:33:33": 44.487894416524036,
     "Emphasis on Accuracy 60:20:20": 38.31837801553863,
     "Emphasis on Cost 20:20:60": 45.2973332743533,
     "Emphasis on Speed 20:60:20": 51.919511056412496,
     "Intelligence only": 31.720043803902097
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 76,
   "ranked": true,
   "ranks": {
    "A": 76,
    "B": 79,
    "C": 84,
    "capability": 116
   },
   "repo": "https://huggingface.co/crh225/plumb-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 47.92548318441584,
    "B": 43.377424184248476,
    "C": 33.28158554473321
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1921826101373881,
    "p50_s_raw": 0.021091305068694055,
    "p95_s_adjusted": 0.22252720159012823,
    "p95_s_raw": 0.03626360079506411
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.6386296587994,
      88.25546310062707
     ],
     "I": [
      37.12848978026168,
      47.77693524356761
     ],
     "capability": [
      60.13168890043501,
      67.144764362578
     ],
     "composite": {
      "A": [
       33.81211971290175,
       61.75277872389874
      ],
      "B": [
       29.947742194356437,
       57.08354879011454
      ],
      "C": [
       23.480638689515107,
       42.88387411381857
      ]
     },
     "seed": 85
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.80311208802716,
     "capability": 65.3199781157169,
     "intelligence": 42.83684414340664
    },
    "per_type": {
     "choice": {
      "accuracy": 0.684,
      "calibration": 87.41108609139918,
      "cc": 59.58811325938065,
      "cc_by_type": {
       "choice": 59.58811325938065
      },
      "cc_tiered": 60.556981688945314,
      "cc_tiered_open": 62.95060504058115,
      "cc_tiered_sealed": 58.16335833730948,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.58811325938065
     },
     "noul": {
      "accuracy": 0.6453333333333333,
      "calibration": 86.76430762472252,
      "cc": 29.066666666666663,
      "cc_by_type": {
       "noul": 29.066666666666663
      },
      "cc_tiered": 29.74780145974502,
      "cc_tiered_open": 36.11249137336094,
      "cc_tiered_sealed": 23.383111546129097,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 29.066666666666663
     },
     "score": {
      "accuracy": 0.576,
      "calibration": 89.2339425479598,
      "cc": 41.48168877172061,
      "cc_by_type": {
       "score": 41.48168877172061
      },
      "cc_tiered": 38.20574928152958,
      "cc_tiered_open": 35.64297548134833,
      "cc_tiered_sealed": 40.76852308171083,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.48168877172061
     }
    },
    "raw_on_A": {
     "calibration": 82.08788861322404,
     "capability": 66.01184998894868,
     "intelligence": 49.93581136467332
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.002666666666666706,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 44.1524794474833,
    "Emphasis on Accuracy 60:20:20": 38.006855950287665,
    "Emphasis on Cost 20:20:60": 44.985580438857504,
    "Emphasis on Speed 20:60:20": 51.530543943271056,
    "Intelligence only": 31.442161631528652
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SPX AI team (SurdAI, formerly Simplex AI)",
   "axes": {
    "calibration": 92.33141939321372,
    "cost": 43.076275488494026,
    "intelligence": 64.6277726176258,
    "speed": 77.43431239288299
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.037067490257276604,
      "ece_score": 92.58650194854468,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.58650194854468,
      "score_before_time_limit": 92.58650194854468,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.1617250673854448,
      "brier": 0.07013566233777668,
      "ece": 0.037583902439496455,
      "n": 371,
      "score": 92.48321951210072,
      "score_before_time_limit": 92.48321951210072,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.983739837398374,
      "interval90_width": 0.26445347786811196,
      "n": 369,
      "nrps": 0.05200753435023827,
      "score": 91.92453671899578,
      "score_before_time_limit": 92.17032425033,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.052292990321580954
     }
    },
    "score": 92.33141939321372
   },
   "capability": 78.47959600541975,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": [
     46.27333224964364,
     48.08682307157107
    ],
    "B": [
     45.84268883284009,
     48.63626352048142
    ],
    "C": [
     45.367953339258825,
     48.08682307157107
    ]
   },
   "cost": {
    "basis": "Operator's own public-beta overage tariff, conservative reading USD 0.08 per 1M input tokens, output USD 0: three dated primary readings of the operator's own published overage tariff: 24 Sep 2026 provider console (flash 0.025, pro 0.08 per 1M input), 1 Oct 2026 docs page (flash 0.04, pro 0.02), 6 Oct 2026 docs page AND /v1/models payload agreeing (flash 0.02, pro 0.04). Sources conflict, so the per-model MAXIMUM is used, which is the published fastino-gliner-2-5-decide precedent (\"published sources conflict ... conservative higher\"). Today's agreeing pair would be USD 0.04/M, i.e. half this row's cost. No base-model floor applies: the submitter states the base model stays sealed until a later open release. Priced on usage.input_tokens as the provider reports them. The provider also reports billable_input_tokens = input_tokens / 2 on every single item (297/297 of the public set, exactly 2.0): its default effort 2 renders the prompt twice and its docs bundle the second pass free during public beta (\"default effort 2 is included at the single-pass input price\"). That bundle is a public-beta promotion, and TASK.md forbids a free or promotional tier setting the price, so the two passes actually consumed are charged. The operator's own billed figure is exactly half of this row's cost",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.07775829333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.07896915368991199
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SPX-CD Pro (SurdAI, hosted /v1/systemone, Oct-4 checkpoint)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows) under Florian's 5 Oct 2026 full-set amendment, same lane, same adapter and same provider default as spx-cd-flash; add-requests run 53, benchmarkheaven.com/submit ref 3FF00545",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 65.26225762662179,
    "I_sealed": 63.993287608629814,
    "base": 64.6277726176258,
    "excess": -1.3044176244462684,
    "gap": 1.2689700179919754,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 77.44982804388046,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 79.23875432525952,
       "judge": 80.22412656558997,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 58.726017943409246,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 19.999999999999996,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 59.61092689257566,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.50227372429029,
       "hard": 68.13445392047002,
       "judge": 54.627278742003114,
       "standard": 48.59367164678844
      }
     },
     "sealed|choice": {
      "cc": 78.81397986232295,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 74.5475113122172,
       "judge": 77.6536312849162,
       "standard": 87.80638009607095
      }
     },
     "sealed|noul": {
      "cc": 53.70130193527029,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 37.86407766990292,
       "judge": 52.63157894736843,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 59.46458102829621,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.86467136136848,
       "hard": 54.98115972463263,
       "judge": 60.73406209967376,
       "standard": 62.82715686202089
      }
     }
    }
   },
   "jevbench_score": 47.553774484141584,
   "key": "spx-cd-pro",
   "last_measured_on": "2026-10-06",
   "licence": null,
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "provider-reported measurement model slug: spx-cd-pro",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9517684887459807,
    "decisive_rate": 0.8382749326145552,
    "supported": true,
    "valid": 371
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 25,
    "axes": {
     "calibration": 92.41334857032513,
     "cost": 43.076275488494026,
     "intelligence": 64.85693822910903,
     "speed": 77.96292698098645
    },
    "board_release": "v1.6.43",
    "capability": 78.63514339971708,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      46.40578610997944,
      48.21089390268207
     ],
     "B": [
      45.98130956469153,
      48.76321124599849
     ],
     "C": [
      45.82343084108345,
      48.21089390268207
     ]
    },
    "jevbench_score": 47.66970820706367,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 68,
    "ranks": {
     "A": 68,
     "B": 65,
     "C": 55,
     "capability": 32
    },
    "scores": {
     "A": 47.66970820706367,
     "B": 47.762738448829694,
     "C": 47.66970820706367
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.1281277425587177,
     "p50_s_raw": 1.1281277425587177,
     "p95_s_adjusted": 1.4169320456683636,
     "p95_s_raw": 1.4169320456683636
    },
    "views": {
     "Balanced 33:33:33": 43.27025827744028,
     "Emphasis on Accuracy 60:20:20": 45.09442248470386,
     "Emphasis on Cost 20:20:60": 37.91159972444074,
     "Emphasis on Speed 20:60:20": 48.12587216483719,
     "Intelligence only": 48.138519063832746
    }
   },
   "provenance": {
    "carried_metadata_from": "Current-pool API measurement registry",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 77,
   "ranked": true,
   "ranks": {
    "A": 77,
    "B": 73,
    "C": 62,
    "capability": 39
   },
   "repo": "https://surd.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 47.553774484141584,
    "B": 47.6361303647083,
    "C": 47.553774484141584
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.1453867703676224,
    "p50_s_raw": 1.1453867703676224,
    "p95_s_adjusted": 1.576219144836068,
    "p95_s_raw": 1.576219144836068
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.37372529307686,
      92.22160309760052
     ],
     "I": [
      59.366080391588525,
      68.65344947022643
     ],
     "capability": [
      74.24700490531005,
      79.69851586001273
     ],
     "composite": {
      "A": [
       46.27333224964364,
       48.08682307157107
      ],
      "B": [
       45.84268883284009,
       48.63626352048142
      ],
      "C": [
       45.367953339258825,
       48.08682307157107
      ]
     },
     "seed": 138
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.33141939321372,
     "capability": 78.47959600541975,
     "intelligence": 64.6277726176258
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8373333333333334,
      "calibration": 92.58650194854468,
      "cc": 79.1972566145335,
      "cc_by_type": {
       "choice": 79.1972566145335
      },
      "cc_tiered": 78.1319039531017,
      "cc_tiered_open": 77.44982804388046,
      "cc_tiered_sealed": 78.81397986232295,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 79.1972566145335
     },
     "noul": {
      "accuracy": 0.7893333333333333,
      "calibration": 92.48321951210072,
      "cc": 57.86666666666667,
      "cc_by_type": {
       "noul": 57.86666666666667
      },
      "cc_tiered": 56.21365993933976,
      "cc_tiered_open": 58.726017943409246,
      "cc_tiered_sealed": 53.70130193527029,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 57.86666666666667
     },
     "score": {
      "accuracy": 0.6826666666666666,
      "calibration": 91.92453671899578,
      "cc": 59.94377608650918,
      "cc_by_type": {
       "score": 59.94377608650918
      },
      "cc_tiered": 59.537753960435936,
      "cc_tiered_open": 59.61092689257566,
      "cc_tiered_sealed": 59.46458102829621,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 59.94377608650918
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 43.15098842304451,
    "Emphasis on Accuracy 60:20:20": 44.95698890795167,
    "Emphasis on Cost 20:20:60": 37.85659305616461,
    "Emphasis on Speed 20:60:20": 47.92862089900984,
    "Intelligence only": 47.968426341937814
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Messier AI (@agentmessier-ai, agentmessier.ai@gmail.com)",
   "axes": {
    "calibration": 73.90972729526267,
    "cost": 64.78240136302922,
    "intelligence": 42.35139759222662,
    "speed": 92.56000105620268
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16098163180524172,
      "ece_score": 67.80367363895165,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 67.80367363895165,
      "score_before_time_limit": 67.80367363895165,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.15200000000000002,
      "brier": 0.14568508389123155,
      "ece": 0.10030106317015194,
      "n": 375,
      "score": 79.93978736596961,
      "score_before_time_limit": 79.93978736596961,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8533333333333334,
      "interval90_width": 0.378888888888889,
      "n": 375,
      "nrps": 0.12363172157496395,
      "score": 73.98572088086677,
      "score_before_time_limit": 73.98572088086677,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1983269304038504
     }
    },
    "score": 73.90972729526267
   },
   "capability": 58.130562443744644,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     29.534709218735255,
     57.65192459942999
    ],
    "B": [
     26.194683217379804,
     53.33943585369998
    ],
    "C": [
     20.510214735232815,
     40.03605874960415
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights). NO JUDGEMENT OF OURS: pricing_v15.reference() resolves Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the frozen 25 Sep snapshot (the same rate aplomb-1 carries for the same base) x this row's own measured tokens. Estimated, not charged. THE FRONT END REPORTS NO TOKEN USAGE (usage {}), exactly as v0.1 did in run 54, so the counts come from run 54's pre-registered route unchanged (PREREG-R59-AMENDMENT-1 = R54-AMENDMENT-2): the author's own prompt.Prompter (v0.2 source, unmodified) recomputes each item's prompt token ids, output is fixed at 1 by their own max_tokens=1, and the totals were required to match vLLM's own counters or the row would get no cost: 914,665 input and 1,500 output tokens both ways (vllm prompt_tokens_total 915,349 - 684; generation_tokens_total 1,508 - 8). Counts sit in cost_estimate, never in usage.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0184433
    },
    "kind": "estimated",
    "usage_estimated_rows": 1500,
    "usd_per_1000": 0.014925192958700068
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Messier One v0.2 (Qwen3.5-4B fine-tune, one prefill, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. REPLACES the v0.1 request (GitHub #194, measured by run 54); the submission names it so. Weights agentmessier/messier-one tag v0.2 = 8bbddb84140e8fc732b35713f998ed628fc70306 (CC BY-NC 4.0, NON-COMMERCIAL), front end github.com/agentmessier-ai/messier-one tag v0.2 = 1b8c3b80 (Apache-2.0). README install line (vllm==0.30.0, transformers==5.18.0; torch 2.13.0+cu130 resolved by vllm) and the author's two serve commands verbatim; --gate off, their default and the configuration of their own reported single-read run. The author discloses that part of the training material was written in the families of JevBench's hard tier, with no JevBench item used. Their Prompter caps the state at 8,000 tokens (max_total 8,600) and splices long states around a readable marker, so the 23 long items are answered truncated (r54 s on disclosed truncation). UNCHANGED typesafe adapter; the smoke gate refused the first attempt at usage-input-tokens and passed with its existing pre-registered opt-in. One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 455A3944",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 42.671434486238354,
    "I_sealed": 42.03136069821489,
    "base": 42.35139759222662,
    "excess": -1.9333138544147794,
    "gap": 0.6400737880234644,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 54.85241164824772,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 60.36489462094999,
       "judge": 36.71720500988794,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 42.58109040717736,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 0.0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 30.580801403290003,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.135231705449634,
       "hard": 32.351343308274984,
       "judge": 27.483794440784738,
       "standard": 22.40801288599812
      }
     },
     "sealed|choice": {
      "cc": 55.440313317505925,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 73.3944276106718,
       "hard": 51.64027149321266,
       "judge": 54.37616387337058,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 41.10238724117829,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 22.330097087378633,
       "judge": 44.736842105263165,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 29.55138153596046,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.40617006718976,
       "hard": 32.98321832976057,
       "judge": 19.789254292160475,
       "standard": 18.90350454844556
      }
     }
    }
   },
   "jevbench_score": 45.27817973469314,
   "key": "messier-one-v0.2",
   "last_measured_on": "2026-10-07",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "agentmessier/messier-one@8bbddb84140e8fc732b35713f998ed628fc70306",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8522012578616353,
    "decisive_rate": 0.848,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.90972729526267,
     "cost": 64.78240136302922,
     "intelligence": 42.35139759222662,
     "speed": 92.70909408602279
    },
    "board_release": "v1.6.43",
    "capability": 58.130562443744644,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 65,
    "capability_rank": 90,
    "composite_ci95": {
     "A": [
      29.542188267758267,
      57.66843444819182
     ],
     "B": [
      26.199389362563995,
      53.35068096010408
     ],
     "C": [
      20.51540851927657,
      40.047523922355424
     ]
    },
    "jevbench_score": 45.29059491870636,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 45,
    "rank": 70,
    "ranks": {
     "A": 70,
     "B": 74,
     "C": 74,
     "capability": 143
    },
    "scores": {
     "A": 45.29059491870636,
     "B": 41.2441782613488,
     "C": 31.451802026879417
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20452698896988294,
     "p50_s_raw": 0.027263494484941475,
     "p95_s_adjusted": 0.26202333298570013,
     "p95_s_raw": 0.056011666492850054
    },
    "views": {
     "Balanced 33:33:33": 43.19017554763209,
     "Emphasis on Accuracy 60:20:20": 36.959944674970394,
     "Emphasis on Cost 20:20:60": 44.448060509071624,
     "Emphasis on Speed 20:60:20": 50.236730867619606,
     "Intelligence only": 30.38527918499109
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 78,
   "ranked": true,
   "ranks": {
    "A": 78,
    "B": 85,
    "C": 85,
    "capability": 161
   },
   "repo": "https://huggingface.co/agentmessier/messier-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 45.27817973469314,
    "B": 41.235940964295494,
    "C": 31.443180371314682
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20892030798713676,
    "p50_s_raw": 0.029460153993568383,
    "p95_s_adjusted": 0.26547231497825124,
    "p95_s_raw": 0.05773615748912562
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.69942429480604,
      76.95757379447907
     ],
     "I": [
      35.53670432007418,
      46.90962732194912
     ],
     "capability": [
      53.17607857615277,
      61.1523915096487
     ],
     "composite": {
      "A": [
       29.534709218735255,
       57.65192459942999
      ],
      "B": [
       26.194683217379804,
       53.33943585369998
      ],
      "C": [
       20.510214735232815,
       40.03605874960415
      ]
     },
     "seed": 93
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.90972729526267,
     "capability": 58.130562443744644,
     "intelligence": 42.35139759222662
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6586666666666666,
      "calibration": 67.80367363895165,
      "cc": 56.348341748529286,
      "cc_by_type": {
       "choice": 56.348341748529286
      },
      "cc_tiered": 55.146362482876825,
      "cc_tiered_open": 54.85241164824772,
      "cc_tiered_sealed": 55.440313317505925,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.348341748529286
     },
     "noul": {
      "accuracy": 0.7226666666666667,
      "calibration": 79.93978736596961,
      "cc": 44.53333333333334,
      "cc_by_type": {
       "noul": 44.53333333333334
      },
      "cc_tiered": 41.841738824177824,
      "cc_tiered_open": 42.58109040717736,
      "cc_tiered_sealed": 41.10238724117829,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 44.53333333333334
     },
     "score": {
      "accuracy": 0.504,
      "calibration": 73.98572088086677,
      "cc": 32.623848126624935,
      "cc_by_type": {
       "score": 32.623848126624935
      },
      "cc_tiered": 30.06609146962523,
      "cc_tiered_open": 30.580801403290003,
      "cc_tiered_sealed": 29.55138153596046,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.623848126624935
     }
    },
    "raw_on_A": {
     "calibration": 79.87142853291357,
     "capability": 63.87968579812095,
     "intelligence": 47.887943063328336
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 43.17512288027295,
    "Emphasis on Accuracy 60:20:20": 36.95332965663756,
    "Emphasis on Cost 20:20:60": 44.43849389377311,
    "Emphasis on Speed 20:60:20": 50.200087678513626,
    "Intelligence only": 30.38527918499109
   }
  },
  {
   "adapter_id": "images/spark-s1-4b-v6/spec.json:spark_v15.Spark",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Abhishek Rai (abhishek085)",
   "axes": {
    "calibration": 64.79562625454915,
    "cost": 60.422063196854964,
    "intelligence": 43.2364603010071,
    "speed": 86.33664750658791
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20104071779836963,
      "ece_score": 59.79185644032607,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 59.79185644032607,
      "score_before_time_limit": 59.79185644032607,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.16000000000000003,
      "brier": 0.16425289707950683,
      "ece": 0.1444944938079116,
      "n": 375,
      "score": 71.10110123841768,
      "score_before_time_limit": 71.10110123841768,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.712,
      "interval90_width": 0.17133333333333342,
      "n": 375,
      "nrps": 0.1542624452699586,
      "score": 63.493921084903704,
      "score_before_time_limit": 63.493921084903704,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.2879295665159836
     }
    },
    "score": 64.79562625454915
   },
   "capability": 54.016043277778124,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     29.741801165157856,
     55.31954608705695
    ],
    "B": [
     26.899062006134436,
     51.98470831594469
    ],
    "C": [
     20.65402858691517,
     38.41635144934511
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020857610837438423
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "spark-s1-4b-v6 (Open Spark Jev, abhishek085)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 47.64157364296756,
    "I_sealed": 38.83134695904664,
    "base": 43.2364603010071,
    "excess": 6.236839041482673,
    "gap": 8.810226683920916,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 63.30038752190768,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 68.35860250494396,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 46.0848861283644,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 33.539447278630604,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 38.99774322763003,
       "hard": 44.04391027813551,
       "judge": 21.602744427042197,
       "standard": 27.706427582503657
      }
     },
     "sealed|choice": {
      "cc": 57.98194607005097,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 50.113122171945705,
       "judge": 60.89385474860336,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 33.87772460002735,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": 2.9126213592232997,
       "judge": 50.0,
       "standard": 60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 24.634370207061615,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 51.84151834556672,
       "hard": 16.688097933910274,
       "judge": 26.27544811817185,
       "standard": 24.46172381744639
      }
     }
    }
   },
   "jevbench_score": 44.84661797633585,
   "key": "spark-s1-4b-v6",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 (code and weights); base Qwen/Qwen3.5-4B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8317460317460318,
    "decisive_rate": 0.84,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 64.79562625454915,
     "cost": 60.422063196854964,
     "intelligence": 43.2364603010071,
     "speed": 86.95618624402076
    },
    "board_release": "v1.6.43",
    "capability": 54.016043277778124,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 86,
    "capability_rank": 112,
    "composite_ci95": {
     "A": [
      29.776291757874876,
      55.39058631138699
     ],
     "B": [
      26.921562908696533,
      52.0349558138311
     ],
     "C": [
      20.677980387413104,
      38.465684938463184
     ]
    },
    "jevbench_score": 44.90217636528639,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 46,
    "rank": 71,
    "ranks": {
     "A": 71,
     "B": 73,
     "C": 75,
     "capability": 167
    },
    "scores": {
     "A": 44.90217636528639,
     "B": 41.66207269525555,
     "C": 31.18206692033776
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.40721648099715824,
     "p50_s_raw": 0.1286082404985791,
     "p95_s_adjusted": 0.4949439602918574,
     "p95_s_raw": 0.17247198014592868
    },
    "views": {
     "Balanced 33:33:33": 43.83191134540322,
     "Emphasis on Accuracy 60:20:20": 38.371611426428935,
     "Emphasis on Cost 20:20:60": 44.36176060417368,
     "Emphasis on Speed 20:60:20": 50.402171180021895,
     "Intelligence only": 32.33034853965725
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 79,
   "ranked": true,
   "ranks": {
    "A": 79,
    "B": 84,
    "C": 87,
    "capability": 187
   },
   "repo": "https://github.com/abhishek085/open-spark-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 44.84661797633585,
    "B": 41.62379682200904,
    "C": 31.14348470578878
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4085616710057366,
    "p50_s_raw": 0.12928083550286829,
    "p95_s_adjusted": 0.5689546199035249,
    "p95_s_raw": 0.20947730995176245
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B with a LoRA, read at the option-letter logits with a fitted temperature (no extra head)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      59.71050928645707,
      68.28479086272019
     ],
     "I": [
      36.48865188869242,
      47.17023357174763
     ],
     "capability": [
      48.85711252205833,
      57.24976166234473
     ],
     "composite": {
      "A": [
       29.741801165157856,
       55.31954608705695
      ],
      "B": [
       26.899062006134436,
       51.98470831594469
      ],
      "C": [
       20.65402858691517,
       38.41635144934511
      ]
     },
     "seed": 102
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 64.79562625454915,
     "capability": 54.016043277778124,
     "intelligence": 43.2364603010071
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 59.79185644032607,
      "cc": 59.92914183947025,
      "cc_by_type": {
       "choice": 59.92914183947025
      },
      "cc_tiered": 60.64116679597932,
      "cc_tiered_open": 63.30038752190768,
      "cc_tiered_sealed": 57.98194607005097,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.92914183947025
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 71.10110123841768,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 39.981305364195876,
      "cc_tiered_open": 46.0848861283644,
      "cc_tiered_sealed": 33.87772460002735,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.5386666666666666,
      "calibration": 63.493921084903704,
      "cc": 27.536544538601227,
      "cc_by_type": {
       "score": 27.536544538601227
      },
      "cc_tiered": 29.08690874284611,
      "cc_tiered_open": 33.539447278630604,
      "cc_tiered_sealed": 24.634370207061615,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.536544538601227
     }
    },
    "raw_on_A": {
     "calibration": 68.7404409420719,
     "capability": 57.96450731310942,
     "intelligence": 47.18857368414694
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 43.76134909822741,
    "Emphasis on Accuracy 60:20:20": 38.339140479599855,
    "Emphasis on Cost 20:20:60": 44.31836606741071,
    "Emphasis on Speed 20:60:20": 50.23451665868595,
    "Intelligence only": 32.33034853965725
   }
  },
  {
   "adapter_id": "images/swanone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "blockbrain",
   "axes": {
    "calibration": 88.58151013990448,
    "cost": 42.15074720483203,
    "intelligence": 56.15104294255194,
    "speed": 82.65355535838808
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03565851721103598,
      "ece_score": 92.8682965577928,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.8682965577928,
      "score_before_time_limit": 92.8682965577928,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.10222826353990752,
      "ece": 0.07688828024001365,
      "n": 371,
      "score": 84.62234395199727,
      "score_before_time_limit": 84.62234395199727,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.924119241192412,
      "interval90_width": 0.2818428184281841,
      "n": 369,
      "nrps": 0.07387893374911293,
      "score": 88.25388990992337,
      "score_before_time_limit": 88.25388990992337,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.08052163402620996
     }
    },
    "score": 88.58151013990448
   },
   "capability": 72.36627654122822,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     42.00043430721653,
     44.67471900154714
    ],
    "B": [
     40.50202737384203,
     44.40460855279977
    ],
    "C": [
     29.166968268900366,
     44.67471900154714
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.08478293103448274
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "swanOne (blockbrain, Qwen3.8-Flash-Next NVFP4)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 56.23071661013901,
    "I_sealed": 56.07136927496486,
    "base": 56.15104294255194,
    "excess": -2.41404030726409,
    "gap": 0.15934733517415367,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 76.36851516458754,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 84.17930125247197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 42.89026915113871,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 39.99999999999999,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 49.43336551469081,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.945051339203516,
       "hard": 45.879570270586335,
       "judge": 54.70148526262065,
       "standard": 44.382933468748654
      }
     },
     "sealed|choice": {
      "cc": 69.57776531270856,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 68.43891402714931,
       "judge": 64.61824953445065,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 45.48583128818902,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 18.446601941747566,
       "judge": 60.52631578947369,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 53.150511223997015,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.92864532337575,
       "hard": 47.07089246791437,
       "judge": 55.79271749917136,
       "standard": 52.457372273711435
      }
     }
    }
   },
   "jevbench_score": 43.78687258347139,
   "key": "swanone",
   "last_measured_on": "2026-10-02",
   "licence": "patches Apache-2.0, shim MIT; weights under the Qwen licence (LICENSE-NOTICE.md)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9269102990033222,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.58151013990448,
     "cost": 42.15074720483203,
     "intelligence": 56.15104294255194,
     "speed": 82.64878475169937
    },
    "board_release": "v1.6.43",
    "capability": 72.36627654122822,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      42.00000021718559,
      44.674228699218794
     ],
     "B": [
      40.50170443761996,
      44.404221038840014
     ],
     "C": [
      29.16666681748999,
      44.674228699218794
     ]
    },
    "jevbench_score": 43.78640157551217,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 47,
    "rank": 72,
    "ranks": {
     "A": 72,
     "B": 71,
     "C": 68,
     "capability": 62
    },
    "scores": {
     "A": 43.78640157551217,
     "B": 42.950891870154436,
     "C": 38.348855694306984
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7114971062168479,
     "p50_s_raw": 0.28074855310842395,
     "p95_s_adjusted": 0.7637450117617846,
     "p95_s_raw": 0.30687250588089227
    },
    "views": {
     "Balanced 33:33:33": 39.75216183669048,
     "Emphasis on Accuracy 60:20:20": 39.813195135198455,
     "Emphasis on Cost 20:20:60": 35.153484373145204,
     "Emphasis on Speed 20:60:20": 45.65457340538371,
     "Intelligence only": 39.90509729608976
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 81,
   "ranked": true,
   "ranks": {
    "A": 81,
    "B": 81,
    "C": 77,
    "capability": 72
   },
   "repo": "https://github.com/blockbrain-ai/swanone-recipe",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 43.78687258347139,
    "B": 42.95125443282615,
    "C": 38.34926821087854
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7102613491937518,
    "p50_s_raw": 0.2801306745968759,
    "p95_s_adjusted": 0.7642338716425001,
    "p95_s_raw": 0.30711693582125005
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.47295009833218,
      89.28636150565644
     ],
     "I": [
      49.957046630174666,
      60.997721736085786
     ],
     "capability": [
      67.47509953288571,
      74.36120880664787
     ],
     "composite": {
      "A": [
       42.00043430721653,
       44.67471900154714
      ],
      "B": [
       40.50202737384203,
       44.40460855279977
      ],
      "C": [
       29.166968268900366,
       44.67471900154714
      ]
     },
     "seed": 104
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.58151013990448,
     "capability": 72.36627654122822,
     "intelligence": 56.15104294255194
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7786666666666666,
      "calibration": 92.8682965577928,
      "cc": 71.69462785256195,
      "cc_by_type": {
       "choice": 71.69462785256195
      },
      "cc_tiered": 72.97314023864806,
      "cc_tiered_open": 76.36851516458754,
      "cc_tiered_sealed": 69.57776531270856,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.69462785256195
     },
     "noul": {
      "accuracy": 0.744,
      "calibration": 84.62234395199727,
      "cc": 48.8,
      "cc_by_type": {
       "noul": 48.8
      },
      "cc_tiered": 44.188050219663864,
      "cc_tiered_open": 42.89026915113871,
      "cc_tiered_sealed": 45.48583128818902,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 48.8
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 88.25388990992337,
      "cc": 52.763382802943624,
      "cc_by_type": {
       "score": 52.763382802943624
      },
      "cc_tiered": 51.29193836934391,
      "cc_tiered_open": 49.43336551469081,
      "cc_tiered_sealed": 53.150511223997015,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.763382802943624
     }
    },
    "raw_on_A": {
     "calibration": 88.23312364196306,
     "capability": 74.40197927194893,
     "intelligence": 60.5708349019348
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 39.75267945658279,
    "Emphasis on Accuracy 60:20:20": 39.81350665991749,
    "Emphasis on Cost 20:20:60": 35.153727243667525,
    "Emphasis on Speed 20:60:20": 45.65580236205991,
    "Intelligence only": 39.90509729608976
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 76.76007761870373,
    "cost": 63.43457194636937,
    "intelligence": 41.73350604436517,
    "speed": 89.14092560273156
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.15434999999999968,
      "ece_score": 69.13000000000007,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 69.13000000000007,
      "score_before_time_limit": 69.13000000000007,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.24,
      "brier": 0.16916629888000007,
      "ece": 0.10974879999999991,
      "n": 375,
      "score": 78.05024000000002,
      "score_before_time_limit": 78.05024000000002,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.4133333333333333,
      "n": 375,
      "nrps": 0.10737614287777777,
      "score": 83.09999285611111,
      "score_before_time_limit": 83.09999285611111,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.11531199999999998
     }
    },
    "score": 76.76007761870373
   },
   "capability": 59.24679183153445,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     30.829815035693155,
     57.195314737994686
    ],
    "B": [
     27.301282886317676,
     52.87778517416482
    ],
    "C": [
     21.409593774786913,
     39.718968568051864
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra Qwen/Qwen3.5-4B list price $0.03/M input (the exact base weights; not on OpenRouter; same reference as reflex-4b, jobe and typecastlm), read 2026-09-24 x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01655189039408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Medium (Chinh Nguyen, Qwen3.5-4B decoder, option-letter logits)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 42.75300136168062,
    "I_sealed": 40.71401072704972,
    "base": 41.73350604436517,
    "excess": -0.5343970078073448,
    "gap": 2.038990634630899,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.4971468014646,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 67.9144385026738,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 16.34230503795721,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 41.41955224562005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.18969696969697,
       "hard": 35.19814814814814,
       "judge": 54.641694915254234,
       "standard": 31.14407407407409
      }
     },
     "sealed|choice": {
      "cc": 59.00724128213771,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 55.20361990950226,
       "judge": 55.30726256983241,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 25.985534052552406,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": 12.621359223300965,
       "judge": 26.315789473684205,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 37.14925684645904,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 55.4977304964539,
       "hard": 32.33151960784312,
       "judge": 40.893173652694614,
       "standard": 31.994619289340076
      }
     }
    }
   },
   "jevbench_score": 43.56144557419446,
   "key": "quyet-1-0-medium",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8421052631578947,
    "decisive_rate": 0.76,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 76.76007761870373,
     "cost": 63.43457194636937,
     "intelligence": 41.73350604436517,
     "speed": 89.46638461924755
    },
    "board_release": "v1.6.43",
    "capability": 59.24679183153445,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 62,
    "capability_rank": 87,
    "composite_ci95": {
     "A": [
      30.84848869900701,
      57.23347326869325
     ],
     "B": [
      27.312995844624446,
      52.90407172237297
     ],
     "C": [
      21.422561596532642,
      39.74546754770365
     ]
    },
    "jevbench_score": 43.58925238992216,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 48,
    "rank": 73,
    "ranks": {
     "A": 73,
     "B": 77,
     "C": 77,
     "capability": 135
    },
    "scores": {
     "A": 43.58925238992216,
     "B": 39.63222739545648,
     "C": 30.270314159668164
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3048233319917927,
     "p50_s_raw": 0.07741166599589633,
     "p95_s_adjusted": 0.3709482546968502,
     "p95_s_raw": 0.1104741273484251
    },
    "views": {
     "Balanced 33:33:33": 41.05875168344406,
     "Emphasis on Accuracy 60:20:20": 35.24739388678317,
     "Emphasis on Cost 20:20:60": 42.25762883934701,
     "Emphasis on Speed 20:60:20": 47.54937088611952,
     "Intelligence only": 29.074657383286326
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 83,
   "ranked": true,
   "ranks": {
    "A": 83,
    "B": 88,
    "C": 89,
    "capability": 154
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Medium",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 43.56144557419446,
    "B": 39.613834296648186,
    "C": 30.251003870968372
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3133888949872926,
    "p50_s_raw": 0.0816944474936463,
    "p95_s_adjusted": 0.38888736818771574,
    "p95_s_raw": 0.11944368409385787
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.2671834460676,
      80.2774031670074
     ],
     "I": [
      36.06269585061453,
      46.7313636734712
     ],
     "capability": [
      54.568687859056396,
      62.84407796698955
     ],
     "composite": {
      "A": [
       30.829815035693155,
       57.195314737994686
      ],
      "B": [
       27.301282886317676,
       52.87778517416482
      ],
      "C": [
       21.409593774786913,
       39.718968568051864
      ]
     },
     "seed": 87
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 76.76007761870373,
     "capability": 59.24679183153445,
     "intelligence": 41.73350604436517
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7013333333333334,
      "calibration": 69.13000000000007,
      "cc": 61.804799029963135,
      "cc_by_type": {
       "choice": 61.804799029963135
      },
      "cc_tiered": 64.75219404180115,
      "cc_tiered_open": 70.4971468014646,
      "cc_tiered_sealed": 59.00724128213771,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.804799029963135
     },
     "noul": {
      "accuracy": 0.64,
      "calibration": 78.05024000000002,
      "cc": 28.000000000000004,
      "cc_by_type": {
       "noul": 28.000000000000004
      },
      "cc_tiered": 21.163919545254807,
      "cc_tiered_open": 16.34230503795721,
      "cc_tiered_sealed": 25.985534052552406,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 28.000000000000004
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 83.09999285611111,
      "cc": 38.22100877192973,
      "cc_by_type": {
       "score": 38.22100877192973
      },
      "cc_tiered": 39.28440454603954,
      "cc_tiered_open": 41.41955224562005,
      "cc_tiered_sealed": 37.14925684645904,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 38.22100877192973
     }
    },
    "raw_on_A": {
     "calibration": 77.01676225205841,
     "capability": 61.22371994730187,
     "intelligence": 45.430677642545334
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 41.025861087818775,
    "Emphasis on Accuracy 60:20:20": 35.23284484921372,
    "Emphasis on Cost 20:20:60": 42.23671880054447,
    "Emphasis on Speed 20:60:20": 47.470039477619395,
    "Intelligence only": 29.074657383286326
   }
  },
  {
   "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/lev.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Interfaze",
   "axes": {
    "calibration": 82.32371666823741,
    "cost": 61.79891447724963,
    "intelligence": 41.084702071234844,
    "speed": 87.52560274254004
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09976787636606844,
      "ece_score": 80.04642472678631,
      "mean_tvd": null,
      "n": 735,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.04642472678631,
      "score_before_time_limit": 80.04642472678631,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.15633423180592987,
      "brier": 0.15547862054178035,
      "ece": 0.09804350768859321,
      "n": 371,
      "score": 80.39129846228136,
      "score_before_time_limit": 80.39129846228136,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.7095754290876237,
      "n": 369,
      "nrps": 0.12033260727134382,
      "score": 86.53342681564456,
      "score_before_time_limit": 86.53342681564456,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07449942820788243
     }
    },
    "score": 82.32371666823741
   },
   "capability": 61.70420936973613,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     28.69291722824612,
     50.68736686039317
    ],
    "B": [
     25.305896361681555,
     46.56027188220375
    ],
    "C": [
     19.92563696405981,
     35.199560319717484
    ]
   },
   "cost": {
    "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3); self-hosted estimate, not measured hosted billing",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01876590517241379
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "lev (Interfaze AI, Qwen3.5-4B + LoRA r32, label-token readout + candidate-path head, per-bucket temperatures)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 45.778585099687206,
    "I_sealed": 36.39081904278248,
    "base": 41.084702071234844,
    "excess": 6.81437841446648,
    "gap": 9.387766056904724,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.009098246344166,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 50.92796476879523,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 56.946169772256724,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 73.91304347826086,
       "judge": 50.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 19.38048728046074,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 22.564349870758527,
       "hard": 19.589685248514364,
       "judge": 26.558897550056017,
       "standard": 6.602544644811692
      }
     },
     "sealed|choice": {
      "cc": 54.52900312820748,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 78.71554208853745,
       "hard": 48.585972850678715,
       "judge": 54.37616387337058,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 29.603595615594433,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 2.9126213592232997,
       "judge": 36.8421052631579,
       "standard": 54.92957746478873
      }
     },
     "sealed|score": {
      "cc": 25.039858384545557,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 47.25762460028372,
       "hard": 16.968805028678304,
       "judge": 35.272820946831395,
       "standard": 14.723638144982221
      }
     }
    }
   },
   "jevbench_score": 42.13701542189482,
   "key": "lev",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "interfaze-ai/lev@7bdc748dffebd85b57ee0dbea8f994c6354fed31 + Abhinavexists/lev@cf104b69329302e4eac674a730c71f3511047db8",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8274760383386581,
    "decisive_rate": 0.8436657681940701,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.32371666823741,
     "cost": 61.79891447724963,
     "intelligence": 41.084702071234844,
     "speed": 88.00251499520694
    },
    "board_release": "v1.6.43",
    "capability": 61.70420936973613,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 55,
    "capability_rank": 80,
    "composite_ci95": {
     "A": [
      28.718912479130207,
      50.73774516604421
     ],
     "B": [
      25.322064520088336,
      46.59426967064348
     ],
     "C": [
      19.943689221618207,
      35.23454525419738
     ]
    },
    "jevbench_score": 42.177760514319104,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 50,
    "rank": 77,
    "ranks": {
     "A": 77,
     "B": 80,
     "C": 81,
     "capability": 124
    },
    "scores": {
     "A": 42.177760514319104,
     "B": 38.201113541708786,
     "C": 29.290111468277157
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3546039809356444,
     "p50_s_raw": 0.10230199046782218,
     "p95_s_adjusted": 0.446688625391107,
     "p95_s_raw": 0.1483443126955535
    },
    "views": {
     "Balanced 33:33:33": 39.03925349464146,
     "Emphasis on Accuracy 60:20:20": 33.569486237063714,
     "Emphasis on Cost 20:20:60": 40.07113476989396,
     "Emphasis on Speed 20:60:20": 45.246505158344306,
     "Intelligence only": 27.739614243661805
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 87,
   "ranked": true,
   "ranks": {
    "A": 87,
    "B": 91,
    "C": 93,
    "capability": 141
   },
   "repo": "https://huggingface.co/interfaze-ai/lev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 42.13701542189482,
    "B": 38.17436710848778,
    "C": 29.26181626520474
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.36095349994720893,
    "p50_s_raw": 0.10547674997360446,
    "p95_s_adjusted": 0.489765814790735,
    "p95_s_raw": 0.16988290739536752
   },
   "status": {
    "answered_ok": 1475,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.63846170548479,
      84.6851724512148
     ],
     "I": [
      35.04763127374306,
      44.44652580243387
     ],
     "capability": [
      56.747890795417284,
      63.66051033763406
     ],
     "composite": {
      "A": [
       28.69291722824612,
       50.68736686039317
      ],
      "B": [
       25.305896361681555,
       46.56027188220375
      ],
      "C": [
       19.92563696405981,
       35.199560319717484
      ]
     },
     "seed": 59
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.32371666823741,
     "capability": 61.70420936973613,
     "intelligence": 41.084702071234844
    },
    "per_type": {
     "choice": {
      "accuracy": 0.664,
      "calibration": 80.04642472678631,
      "cc": 57.030398908708534,
      "cc_by_type": {
       "choice": 57.030398908708534
      },
      "cc_tiered": 57.76905068727582,
      "cc_tiered_open": 61.009098246344166,
      "cc_tiered_sealed": 54.52900312820748,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.030398908708534
     },
     "noul": {
      "accuracy": 0.6906666666666667,
      "calibration": 80.39129846228136,
      "cc": 38.13333333333333,
      "cc_by_type": {
       "noul": 38.13333333333333
      },
      "cc_tiered": 43.274882693925576,
      "cc_tiered_open": 56.946169772256724,
      "cc_tiered_sealed": 29.603595615594433,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 38.13333333333333
     },
     "score": {
      "accuracy": 0.4533333333333333,
      "calibration": 86.53342681564456,
      "cc": 23.66953381268505,
      "cc_by_type": {
       "score": 23.66953381268505
      },
      "cc_tiered": 22.210172832503147,
      "cc_tiered_open": 19.38048728046074,
      "cc_tiered_sealed": 25.039858384545557,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.66953381268505
     }
    },
    "raw_on_A": {
     "calibration": 79.50973335881918,
     "capability": 61.67507354014826,
     "intelligence": 43.84041372147735
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.020000000000000018,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 38.99272148510723,
    "Emphasis on Accuracy 60:20:20": 33.548830528994046,
    "Emphasis on Cost 20:20:60": 40.04170666958352,
    "Emphasis on Speed 20:60:20": 45.134140566564014,
    "Intelligence only": 27.739614243661805
   }
  },
  {
   "adapter_id": "images/ninfer-qwen3.8-flash-next/spec.json:ninfer_native.NInferNativeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Igor L. / NInfer contributors",
   "axes": {
    "calibration": 90.37232453477458,
    "cost": 42.53362878447125,
    "intelligence": 48.980301026876646,
    "speed": 89.03327048907914
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03727735037984102,
      "ece_score": 92.54452992403179,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.54452992403179,
      "score_before_time_limit": 92.54452992403179,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.24528301886792447,
      "brier": 0.10381961467316755,
      "ece": 0.05486291160058669,
      "n": 371,
      "score": 89.02741767988266,
      "score_before_time_limit": 89.02741767988266,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.94579945799458,
      "interval90_width": 0.40876242095754284,
      "n": 369,
      "nrps": 0.08645101367263491,
      "score": 89.5450260004093,
      "score_before_time_limit": 89.5450260004093,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06132423315958956
     }
    },
    "score": 90.37232453477458
   },
   "capability": 69.67631278082561,
   "class": "native-logit",
   "composite_ci95": {
    "A": [
     30.258741238230442,
     44.79382690643278
    ],
    "B": [
     28.308012634250343,
     43.47722853150174
    ],
    "C": [
     21.01301474877114,
     36.201102310973276
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.08232764778325123
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "NInfer Qwen3.8-Flash-Next mixed",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 48.827164297755125,
    "I_sealed": 49.13343775599816,
    "base": 48.980301026876646,
    "excess": -2.8796611006812776,
    "gap": -0.30627345824303376,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.22063841159643,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 60.36489462094999,
       "judge": 76.26895187870797,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 36.42788129744652,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": 19.999999999999996,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 39.83297318422244,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.934842929432264,
       "hard": 29.854097365310107,
       "judge": 50.77541876771969,
       "standard": 31.326121574196307
      }
     },
     "sealed|choice": {
      "cc": 68.53578282889805,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 61.82126696832577,
       "judge": 68.34264432029796,
       "standard": 77.82978199285628
      }
     },
     "sealed|noul": {
      "cc": 34.31380022598546,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": -0.9708737864077666,
       "judge": 52.63157894736843,
       "standard": 60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 44.55073021311098,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.34071381045655,
       "hard": 36.06656765199164,
       "judge": 48.53812828291905,
       "standard": 44.64296643196477
      }
     }
    }
   },
   "jevbench_score": 41.944069528166864,
   "key": "ninfer-qwen3.8-flash-next",
   "last_measured_on": "2026-10-02",
   "licence": "Qwen Community License 1.0 (model); Apache-2.0 (engine)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9321428571428572,
    "decisive_rate": 0.7547169811320755,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.37232453477458,
     "cost": 42.53362878447125,
     "intelligence": 48.980301026876646,
     "speed": 89.33173525631216
    },
    "board_release": "v1.6.43",
    "capability": 69.67631278082561,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      30.27504047421702,
      44.819854782587115
     ],
     "B": [
      28.319423416872812,
      43.496842238713214
     ],
     "C": [
      21.02433366265071,
      36.222113908221345
     ]
    },
    "jevbench_score": 41.96785076957165,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 51,
    "rank": 78,
    "ranks": {
     "A": 78,
     "B": 75,
     "C": 82,
     "capability": 81
    },
    "scores": {
     "A": 41.96785076957165,
     "B": 40.09260225022487,
     "C": 29.144340812202532
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3100576108321548,
     "p50_s_raw": 0.08002880541607738,
     "p95_s_adjusted": 0.37616993160918355,
     "p95_s_raw": 0.11308496580459178
    },
    "views": {
     "Balanced 33:33:33": 37.79450787350779,
     "Emphasis on Accuracy 60:20:20": 36.18545094340192,
     "Emphasis on Cost 20:20:60": 33.99296424621917,
     "Emphasis on Speed 20:60:20": 44.79615638245711,
     "Intelligence only": 34.01333411780027
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 88,
   "ranked": true,
   "ranks": {
    "A": 88,
    "B": 86,
    "C": 94,
    "capability": 95
   },
   "repo": "https://github.com/igorls/ninfer",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 41.944069528166864,
    "B": 40.07523713932242,
    "C": 29.127826061226987
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.31898033767938616,
    "p50_s_raw": 0.08449016883969307,
    "p95_s_adjusted": 0.39165983591228726,
    "p95_s_raw": 0.12082991795614362
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-Flash-Next, mixed NVFP4/FP8/INT4 NInfer artifact",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.49584722295648,
      90.3923314283596
     ],
     "I": [
      42.680473865516596,
      53.996834913811625
     ],
     "capability": [
      64.4871361894938,
      71.33725830271337
     ],
     "composite": {
      "A": [
       30.258741238230442,
       44.79382690643278
      ],
      "B": [
       28.308012634250343,
       43.47722853150174
      ],
      "C": [
       21.01301474877114,
       36.201102310973276
      ]
     },
     "seed": 73
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.37232453477458,
     "capability": 69.67631278082561,
     "intelligence": 48.980301026876646
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7613333333333333,
      "calibration": 92.54452992403179,
      "cc": 69.47794208197946,
      "cc_by_type": {
       "choice": 69.47794208197946
      },
      "cc_tiered": 69.37821062024724,
      "cc_tiered_open": 70.22063841159643,
      "cc_tiered_sealed": 68.53578282889805,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.47794208197946
     },
     "noul": {
      "accuracy": 0.696,
      "calibration": 89.02741767988266,
      "cc": 39.19999999999999,
      "cc_by_type": {
       "noul": 39.19999999999999
      },
      "cc_tiered": 35.37084076171599,
      "cc_tiered_open": 36.42788129744652,
      "cc_tiered_sealed": 34.31380022598546,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.19999999999999
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 89.5450260004093,
      "cc": 43.98717489610087,
      "cc_by_type": {
       "score": 43.98717489610087
      },
      "cc_tiered": 42.19185169866671,
      "cc_tiered_open": 39.83297318422244,
      "cc_tiered_sealed": 44.55073021311098,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.98717489610087
     }
    },
    "raw_on_A": {
     "calibration": 86.91372207226564,
     "capability": 70.43814133335954,
     "intelligence": 53.962560594453436
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 37.76879516714813,
    "Emphasis on Accuracy 60:20:20": 36.17130488716136,
    "Emphasis on Cost 20:20:60": 33.98048018861018,
    "Emphasis on Speed 20:60:20": 44.73118686468517,
    "Intelligence only": 34.01333411780027
   }
  },
  {
   "adapter_id": "images/jev-local/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "us (GitHub)",
   "axes": {
    "calibration": 73.06634984647151,
    "cost": 58.85554343293177,
    "intelligence": 41.85041350559534,
    "speed": 72.89461160481758
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16559413680781765,
      "ece_score": 66.88117263843647,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 66.88117263843647,
      "score_before_time_limit": 66.88117263843647,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.14555256064690025,
      "brier": 0.16930220654986547,
      "ece": 0.14746549865229122,
      "n": 371,
      "score": 70.50690026954176,
      "score_before_time_limit": 70.50690026954176,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8644986449864499,
      "interval90_width": 0.325880758807588,
      "n": 369,
      "nrps": 0.10296637523035226,
      "score": 81.81097663143632,
      "score_before_time_limit": 81.81097663143632,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1304070460704606
     }
    },
    "score": 73.06634984647151
   },
   "capability": 57.458381676033426,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     28.9449485360543,
     52.520885216103025
    ],
    "B": [
     26.188968315954813,
     49.615608561621414
    ],
    "C": [
     20.100658705593265,
     36.472836955627095
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.023522413793103446
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jev-local (Qwen3.5-9B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 43.39498650106927,
    "I_sealed": 40.30584051012142,
    "base": 41.85041350559534,
    "excess": 0.5157583485096047,
    "gap": 3.0891459909478485,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 56.18661240978703,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 49.040578798364265,
       "judge": 52.537903757415954,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 34.189786059351285,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 30.000000000000004,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 39.8085610340695,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 40.41969696969697,
       "hard": 51.80083333333333,
       "judge": 22.97271186440677,
       "standard": 40.77222222222221
      }
     },
     "sealed|choice": {
      "cc": 47.63585353447873,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 58.7613627965413,
       "hard": 39.42307692307691,
       "judge": 48.78957169459964,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 35.679356877082625,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 14.563106796116498,
       "judge": 42.10526315789473,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 37.602311118802916,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.23531914893618,
       "hard": 29.158455882352953,
       "judge": 39.61335329341317,
       "standard": 38.15695431472081
      }
     }
    }
   },
   "jevbench_score": 41.03574692794271,
   "key": "jev-local",
   "last_measured_on": "2026-10-02",
   "licence": "no licence stated in the repository (public code); Apache-2.0 base weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8233438485804416,
    "decisive_rate": 0.8544474393530997,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.06634984647151,
     "cost": 58.85554343293177,
     "intelligence": 41.85041350559534,
     "speed": 75.55955847735143
    },
    "board_release": "v1.6.43",
    "capability": 57.458381676033426,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 67,
    "capability_rank": 92,
    "composite_ci95": {
     "A": [
      29.136540211681222,
      52.90884989606441
     ],
     "B": [
      26.3171992505717,
      49.89250384145016
     ],
     "C": [
      20.233708480334183,
      36.74225687226694
     ]
    },
    "jevbench_score": 41.328564334669615,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 52,
    "rank": 79,
    "ranks": {
     "A": 79,
     "B": 81,
     "C": 83,
     "capability": 146
    },
    "scores": {
     "A": 41.328564334669615,
     "B": 38.19938992160457,
     "C": 28.700391899076113
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.8806604199111462,
     "p50_s_raw": 0.3653302099555731,
     "p95_s_adjusted": 3.156717182928696,
     "p95_s_raw": 1.5033585914643481
    },
    "views": {
     "Balanced 33:33:33": 38.834988968034565,
     "Emphasis on Accuracy 60:20:20": 34.37288631458028,
     "Emphasis on Cost 20:20:60": 39.75999364263786,
     "Emphasis on Speed 20:60:20": 43.46632094435822,
     "Intelligence only": 29.31968172619126
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 92,
   "ranked": true,
   "ranks": {
    "A": 92,
    "B": 92,
    "C": 97,
    "capability": 163
   },
   "repo": "https://github.com/us/jev-local",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 41.03574692794271,
    "B": 37.998895811484516,
    "C": 28.497046477737985
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.0924282498657703,
    "p50_s_raw": 0.47121412493288517,
    "p95_s_adjusted": 4.700520133133978,
    "p95_s_raw": 2.2752600665669886
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B, frozen, per-option mean log-probability",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.57161963107492,
      76.81314888025398
     ],
     "I": [
      36.13732889468463,
      46.54627874317067
     ],
     "capability": [
      53.19292305173127,
      60.44357521274037
     ],
     "composite": {
      "A": [
       28.9449485360543,
       52.520885216103025
      ],
      "B": [
       26.188968315954813,
       49.615608561621414
      ],
      "C": [
       20.100658705593265,
       36.472836955627095
      ]
     },
     "seed": 45
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.06634984647151,
     "capability": 57.458381676033426,
     "intelligence": 41.85041350559534
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6026666666666667,
      "calibration": 66.88117263843647,
      "cc": 49.186741566647385,
      "cc_by_type": {
       "choice": 49.186741566647385
      },
      "cc_tiered": 51.91123297213288,
      "cc_tiered_open": 56.18661240978703,
      "cc_tiered_sealed": 47.63585353447873,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.186741566647385
     },
     "noul": {
      "accuracy": 0.696,
      "calibration": 70.50690026954176,
      "cc": 39.19999999999999,
      "cc_by_type": {
       "noul": 39.19999999999999
      },
      "cc_tiered": 34.93457146821696,
      "cc_tiered_open": 34.189786059351285,
      "cc_tiered_sealed": 35.679356877082625,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.19999999999999
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 81.81097663143632,
      "cc": 39.00943859649125,
      "cc_by_type": {
       "score": 39.00943859649125
      },
      "cc_tiered": 38.70543607643621,
      "cc_tiered_open": 39.8085610340695,
      "cc_tiered_sealed": 37.602311118802916,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.00943859649125
     }
    },
    "raw_on_A": {
     "calibration": 73.6085835138451,
     "capability": 59.88003789504907,
     "intelligence": 46.151492276253045
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 38.49087371737372,
    "Emphasis on Accuracy 60:20:20": 34.21046270227734,
    "Emphasis on Cost 20:20:60": 39.54282942742431,
    "Emphasis on Speed 20:60:20": 42.69728156694438,
    "Intelligence only": 29.31968172619126
   }
  },
  {
   "adapter_id": "images/decider-4b-v2/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 89.63735637833331,
    "cost": 64.54376873042901,
    "intelligence": 40.12597268917964,
    "speed": 90.31631787234045
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04618531250000016,
      "ece_score": 90.76293749999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 90.76293749999998,
      "score_before_time_limit": 90.76293749999998,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3573333333333333,
      "brier": 0.1263556454400002,
      "ece": 0.05272693333333338,
      "n": 375,
      "score": 89.45461333333333,
      "score_before_time_limit": 89.45461333333333,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.5584444444444441,
      "n": 375,
      "nrps": 0.09968243396666672,
      "score": 88.69451830166665,
      "score_before_time_limit": 88.69451830166665,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06321360000000002
     }
    },
    "score": 89.63735637833331
   },
   "capability": 64.88166453375648,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     28.651236758605318,
     49.778681715199255
    ],
    "B": [
     25.06337462145405,
     45.14461375444511
    ],
    "C": [
     19.89669219347592,
     34.56852896888838
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015201077586206898
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-4b v2 (Mapika)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 45.24914817619414,
    "I_sealed": 35.00279720216514,
    "base": 40.12597268917964,
    "excess": 7.672963331590756,
    "gap": 10.246350974029,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 73.21703197407996,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 76.26895187870797,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 26.461697722567287,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 36.06871483193517,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.519696969696966,
       "hard": 36.743888888888875,
       "judge": 38.64050847457627,
       "standard": 24.635185185185193
      }
     },
     "sealed|choice": {
      "cc": 56.802191819905744,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 50.62217194570135,
       "judge": 57.16945996275605,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 11.645451862940547,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": -12.621359223300976,
       "judge": 23.684210526315795,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 36.56074792364913,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 54.016382978723385,
       "hard": 31.53129901960784,
       "judge": 40.6503592814371,
       "standard": 31.75741116751262
      }
     }
    }
   },
   "jevbench_score": 41.12421662138689,
   "key": "decider-4b-v2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (package and weights)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.921161825726141,
    "decisive_rate": 0.6426666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.63735637833331,
     "cost": 64.54376873042901,
     "intelligence": 40.12597268917964,
     "speed": 91.78549329698745
    },
    "board_release": "v1.6.43",
    "capability": 64.88166453375648,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 39,
    "capability_rank": 61,
    "composite_ci95": {
     "A": [
      28.727001690110686,
      49.92424966763833
     ],
     "B": [
      25.10971569724804,
      45.24030143351226
     ],
     "C": [
      19.949306729243542,
      34.66961782474885
     ]
    },
    "jevbench_score": 41.240894350857424,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 53,
    "rank": 80,
    "ranks": {
     "A": 80,
     "B": 83,
     "C": 84,
     "capability": 103
    },
    "scores": {
     "A": 41.240894350857424,
     "B": 36.84955227461344,
     "C": 28.639509965873213
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19992743488401174,
     "p50_s_raw": 0.024963717442005873,
     "p95_s_adjusted": 0.3315723263658583,
     "p95_s_raw": 0.09078616318292916
    },
    "views": {
     "Balanced 33:33:33": 37.65578397809399,
     "Emphasis on Accuracy 60:20:20": 31.83485819738892,
     "Emphasis on Cost 20:20:60": 39.12905962819718,
     "Emphasis on Speed 20:60:20": 44.051943214406094,
     "Intelligence only": 25.842630080539234
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 89,
   "ranked": true,
   "ranks": {
    "A": 89,
    "B": 94,
    "C": 95,
    "capability": 118
   },
   "repo": "https://github.com/Mapika/decider",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 41.12421662138689,
    "B": 36.7749698009312,
    "C": 28.55848376485201
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20652835872024297,
    "p50_s_raw": 0.02826417936012149,
    "p95_s_adjusted": 0.45018240716308344,
    "p95_s_raw": 0.15009120358154174
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.18439184191017,
      90.03628745741203
     ],
     "I": [
      34.68576750629042,
      43.56524556183246
     ],
     "capability": [
      60.38851961863991,
      66.00729141285105
     ],
     "composite": {
      "A": [
       28.651236758605318,
       49.778681715199255
      ],
      "B": [
       25.06337462145405,
       45.14461375444511
      ],
      "C": [
       19.89669219347592,
       34.56852896888838
      ]
     },
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.63735637833331,
     "capability": 64.88166453375648,
     "intelligence": 40.12597268917964
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6933333333333334,
      "calibration": 90.76293749999998,
      "cc": 60.781713289694295,
      "cc_by_type": {
       "choice": 60.781713289694295
      },
      "cc_tiered": 65.00961189699285,
      "cc_tiered_open": 73.21703197407996,
      "cc_tiered_sealed": 56.802191819905744,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.781713289694295
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 89.45461333333333,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 19.053574792753917,
      "cc_tiered_open": 26.461697722567287,
      "cc_tiered_sealed": 11.645451862940547,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.512,
      "calibration": 88.69451830166665,
      "cc": 36.877482456140385,
      "cc_by_type": {
       "score": 36.877482456140385
      },
      "cc_tiered": 36.31473137779215,
      "cc_tiered_open": 36.06871483193517,
      "cc_tiered_sealed": 36.56074792364913,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.877482456140385
     }
    },
    "raw_on_A": {
     "calibration": 88.78997857788461,
     "capability": 66.11763148027418,
     "intelligence": 43.44528438266377
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 37.52616551236565,
    "Emphasis on Accuracy 60:20:20": 31.779178376740063,
    "Emphasis on Cost 20:20:60": 39.04497496232663,
    "Emphasis on Speed 20:60:20": 43.733848636619484,
    "Intelligence only": 25.842630080539234
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 58.71667533779978,
    "cost": 49.33529996402755,
    "intelligence": 42.225671849648506,
    "speed": 89.69784178772274
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.24400358306188902,
      "ece_score": 51.199283387622195,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 51.199283387622195,
      "score_before_time_limit": 51.199283387622195,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.03234501347708896,
      "brier": 0.20490283425876044,
      "ece": 0.20667035040431214,
      "n": 371,
      "score": 58.66592991913757,
      "score_before_time_limit": 58.66592991913757,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.7696476964769647,
      "interval90_width": 0.23373983739837403,
      "n": 369,
      "nrps": 0.141136266192412,
      "score": 66.2848127066396,
      "score_before_time_limit": 66.2848127066396,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.266583739837398
     }
    },
    "score": 58.71667533779978
   },
   "capability": 50.471173593724146,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     24.964213028933468,
     50.894171196974504
    ],
    "B": [
     22.86606015344133,
     48.77975059683428
    ],
    "C": [
     17.336259047870463,
     35.34317444234341
    ]
   },
   "cost": {
    "basis": "ESTIMATE (exact base-model market reference; self-hosted open weights, no per-token tariff of its own). THE ONLY ROW OF THIS RUN WHOSE RATE NEEDED NO JUDGEMENT AT ALL: Qwen/Qwen3.5-9B is in the frozen module's BASE_REFERENCES and pricing_v15.reference() resolves it straight through to qwen/qwen3.5-9b at USD 0.10 per 1M input and USD 0.15 per 1M output in the dated 25 Sep snapshot (DeepInfra's listing of the same base agrees at (0.10, 0.15)) x this row's own measured tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0480964
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04884536222071767
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Qwen 3.5 9B (Qwen3.5-9B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...). Weights devops-thiago/classone-qwen3.5-9b (sha a45d16f1f5d1840a420f8ede59fa5cca3f079961), Apache-2.0, public and ungated; code github.com/devops-thiago/class-one, Apache-2.0. ONE DOCUMENTED MAPPING OF OURS, AND IT IS THE ONLY ONE. The author's server keys SCORE probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1'. The UNAMENDED smoke gate REFUSED the first attempt at check score-keys-are-indices, before any sealed item was sent, and the server was torn down with the card verified free. The 1-based convention - which their own schemas.py documents only as 'Probability distribution across each ordered rubric level', without stating the convention - was then established on the AUTHOR'S OWN SERVER with SYNTHETIC requests across every rubric size their own validator allows (2..10): exactly the keys '1'..'n' in order every time, never '0' and never a key above n. It is corroborated independently by their own `score` field being a 1-based probability-weighted mean of the same distribution (1.0185 on a 2-level rubric, 1.6147 on a 10-level one, 3.1155 where the mass sits on key 3; a 0-based convention would have given 0.0185), and a reversed rubric moved the mass to a different key, so 'k' means the k-th level of the criteria array AS SENT - which is what JevBench's 'k-1' means. classone_reindex_r58.ClassOneReindexAdapter therefore relabels score keys k -> k-1 and NOTHING else: the request body, the answer mapping, the noul range check and the choice-must-be-one-of-the-item's-labels rule are the UNCHANGED vendored typesafe adapter's, reused by calling its own run(). An item whose score key set is not exactly {'1'..'n'} is REFUSED and recorded invalid rather than rescored on a guess; none was. choice and noul are untouched. The gate then ran AMENDED in exactly that one check, requiring the key set to be exactly {'1'..'n'} - stricter than the original, because it also pins the count - with the other seven byte-identical, and passed. PREREG-R58-AMENDMENT-1.md (sha256 in PREREG-R58-AMENDMENT-1.md.sha256), receipt PROBE-classone-e2b.json. No file of the author's was edited and no prompt of ours exists. THE SUBMIT FORM CARRIED NO DESCRIPTION FOR EITHER CLASSONE ROW, so everything about how it is run comes from the author's own repository: install line README Quickstart step 1 (`pip install -e \".[dev]\"`, in an isolated venv - torch 2.14.1+cu130, transformers 5.19.0, fastapi 0.142.2), serve line step 3 (`uvicorn classone.server.app:app`), and the checkpoint selected the way their own src/classone/server/app.py reads it (CLASSONE_BASE_MODEL). THEIR SERVER REGISTERS POST /v1/systemone ITSELF (app.py:115, next to /v1/decide and /v1/classone) and their DecisionRequest/DecisionResponse in src/classone/schemas.py are the TypeSafe System One shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. output_tokens is 0 by their own schema comment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. add-requests run 58, benchmarkheaven.com/submit ref 957CEE0B (bh-submit b45cdc66-df44-4d77-a616-e7c67c83706d)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB",
   "intelligence": {
    "I_open": 42.822420470216805,
    "I_sealed": 41.628923229080215,
    "base": 42.225671849648506,
    "excess": -1.3798904013016533,
    "gap": 1.1934972411365905,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 54.893255386978964,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 49.040578798364265,
       "judge": 36.71720500988794,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 50.0351966873706,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 23.538809336300865,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.58363636363636,
       "hard": 33.53250000000001,
       "judge": 30.56728813559323,
       "standard": -19.51370370370371
      }
     },
     "sealed|choice": {
      "cc": 50.4691947514562,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 39.42307692307691,
       "judge": 51.58286778398511,
       "standard": 63.419140288212844
      }
     },
     "sealed|noul": {
      "cc": 45.96319747817513,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 20.388349514563096,
       "judge": 57.89473684210527,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 28.454377457609333,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.85865248226952,
       "hard": 11.203235294117707,
       "judge": 44.33532934131734,
       "standard": 20.433096446700493
      }
     }
    }
   },
   "jevbench_score": 38.505734160635136,
   "key": "classone-qwen3.5-9b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-qwen3.5-9b@a45d16f1f5d1840a420f8ede59fa5cca3f079961",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7855153203342619,
    "decisive_rate": 0.967654986522911,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 58.71667533779978,
     "cost": 49.33529996402755,
     "intelligence": 42.225671849648506,
     "speed": 90.24509258883624
    },
    "board_release": "v1.6.43",
    "capability": 50.471173593724146,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 90,
    "capability_rank": 117,
    "composite_ci95": {
     "A": [
      24.985716007949218,
      50.94397024296676
     ],
     "B": [
      22.880770578118387,
      48.816339954765404
     ],
     "C": [
      17.35119167218696,
      35.37775711317136
     ]
    },
    "jevbench_score": 38.54185769307265,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 55,
    "rank": 85,
    "ranks": {
     "A": 85,
     "B": 85,
     "C": 86,
     "capability": 172
    },
    "scores": {
     "A": 38.54185769307265,
     "B": 36.26089754980279,
     "C": 26.765178953522675
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2636520860018209,
     "p50_s_raw": 0.05682604300091043,
     "p95_s_adjusted": 0.35847554139327253,
     "p95_s_raw": 0.10423777069663628
    },
    "views": {
     "Balanced 33:33:33": 37.852042064412586,
     "Emphasis on Accuracy 60:20:20": 33.90552600237263,
     "Emphasis on Cost 20:20:60": 36.32702436923972,
     "Emphasis on Speed 20:60:20": 44.975093090995614,
     "Intelligence only": 29.320085596648227
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 97,
   "ranked": true,
   "ranks": {
    "A": 97,
    "B": 97,
    "C": 99,
    "capability": 193
   },
   "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 38.505734160635136,
    "B": 36.23531212076388,
    "C": 26.740093167107737
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27806573198759,
    "p50_s_raw": 0.06403286599379499,
    "p95_s_adjusted": 0.3855390299024293,
    "p95_s_raw": 0.11776951495121465
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      54.01204599443065,
      63.223240175665964
     ],
     "I": [
      35.33390022059835,
      47.469588715973586
     ],
     "capability": [
      45.37052539208047,
      54.754194446307686
     ],
     "composite": {
      "A": [
       24.964213028933468,
       50.894171196974504
      ],
      "B": [
       22.86606015344133,
       48.77975059683428
      ],
      "C": [
       17.336259047870463,
       35.34317444234341
      ]
     },
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 58.71667533779978,
     "capability": 50.471173593724146,
     "intelligence": 42.225671849648506
    },
    "per_type": {
     "choice": {
      "accuracy": 0.624,
      "calibration": 51.199283387622195,
      "cc": 51.9149702073643,
      "cc_by_type": {
       "choice": 51.9149702073643
      },
      "cc_tiered": 52.68122506921758,
      "cc_tiered_open": 54.893255386978964,
      "cc_tiered_sealed": 50.4691947514562,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.9149702073643
     },
     "noul": {
      "accuracy": 0.752,
      "calibration": 58.66592991913757,
      "cc": 50.4,
      "cc_by_type": {
       "noul": 50.4
      },
      "cc_tiered": 47.99919708277287,
      "cc_tiered_open": 50.0351966873706,
      "cc_tiered_sealed": 45.96319747817513,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 50.4
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 66.2848127066396,
      "cc": 27.883929824561417,
      "cc_by_type": {
       "score": 27.883929824561417
      },
      "cc_tiered": 25.9965933969551,
      "cc_tiered_open": 23.538809336300865,
      "cc_tiered_sealed": 28.454377457609333,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.883929824561417
     }
    },
    "raw_on_A": {
     "calibration": 63.67698210538227,
     "capability": 55.155564978213434,
     "intelligence": 46.63414785104459
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 37.805599484143976,
    "Emphasis on Accuracy 60:20:20": 33.88315546161274,
    "Emphasis on Cost 20:20:60": 36.30134557093612,
    "Emphasis on Speed 20:60:20": 44.85723809337993,
    "Intelligence only": 29.320085596648227
   }
  },
  {
   "adapter_id": "images/jevk5-v0.3-4b/spec.json:jevk5_v15.JevK5Warm",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 89.80312361577623,
    "cost": 63.07359395281681,
    "intelligence": 38.53201254374527,
    "speed": 93.8287164149578
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05458773337668626,
      "ece_score": 89.08245332466275,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.84490011579697,
      "score_before_time_limit": 89.08245332466275,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0.33692722371967654,
      "brier": 0.11477714806826486,
      "ece": 0.053786553580231086,
      "n": 371,
      "score": 89.24268928395378,
      "score_before_time_limit": 89.24268928395378,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9566395663956639,
      "interval90_width": 0.4780939476061429,
      "n": 369,
      "nrps": 0.09977232020092458,
      "score": 91.32178144757796,
      "score_before_time_limit": 91.32178144757796,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03689602542375804
     }
    },
    "score": 89.80312361577623
   },
   "capability": 64.16756807976074,
   "class": "unclassified",
   "composite_ci95": {
    "A": [
     24.235506619160738,
     49.5798377283499
    ],
    "B": [
     20.90293459315668,
     45.0236262225029
    ],
    "C": [
     16.83021292997273,
     34.43044286690966
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01701689039408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JevK5 v0.3 (4B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 38.333624208131226,
    "I_sealed": 38.730400879359316,
    "base": 38.53201254374527,
    "excess": -2.970164313666333,
    "gap": -0.39677667122808913,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 60.58350447013268,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 60.36489462094999,
       "judge": 64.40342781806196,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 20.41614906832298,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 47.82608695652173,
       "judge": -9.999999999999998,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 34.00121908593804,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.18739024047374,
       "hard": 34.10450662572488,
       "judge": 37.96632221813109,
       "standard": 18.253903730806933
      }
     },
     "sealed|choice": {
      "cc": 58.82061791071595,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 55.712669683257914,
       "judge": 59.031657355679705,
       "standard": 60.09360758714129
      }
     },
     "sealed|noul": {
      "cc": 22.94594773546748,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": 0.9708737864077666,
       "judge": 39.47368421052631,
       "standard": 29.5774647887324
      }
     },
     "sealed|score": {
      "cc": 34.42463699189452,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.98468618250719,
       "hard": 27.333586499919292,
       "judge": 34.53681846177148,
       "standard": 32.6584411757232
      }
     }
    }
   },
   "jevbench_score": 37.351269273786905,
   "key": "jevk5-v0.3-4b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9471544715447154,
    "decisive_rate": 0.6630727762803235,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.88230801873149,
     "cost": 63.07359395281681,
     "intelligence": 38.53201254374527,
     "speed": 93.9472917853559
    },
    "board_release": "v1.6.43",
    "capability": 64.20716028123837,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 40,
    "capability_rank": 62,
    "composite_ci95": {
     "A": [
      24.2437041451056,
      49.59495089694384
     ],
     "B": [
      20.90887497172086,
      45.03359602914599
     ],
     "C": [
      16.835905656323334,
      34.440938122877675
     ]
    },
    "jevbench_score": 37.36493548751909,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 56,
    "rank": 86,
    "ranks": {
     "A": 86,
     "B": 90,
     "C": 87,
     "capability": 104
    },
    "scores": {
     "A": 37.36493548751909,
     "B": 33.16714732620591,
     "C": 25.947871866332704
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18763320203870534,
     "p50_s_raw": 0.018816601019352674,
     "p95_s_adjusted": 0.21476382557302715,
     "p95_s_raw": 0.03238191278651357
    },
    "views": {
     "Balanced 33:33:33": 33.96793409727579,
     "Emphasis on Accuracy 60:20:20": 28.454799805414766,
     "Emphasis on Cost 20:20:60": 35.283091257812984,
     "Emphasis on Speed 20:60:20": 40.26908605684886,
     "Intelligence only": 22.883638070578947
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 98,
   "ranked": true,
   "ranks": {
    "A": 98,
    "B": 102,
    "C": 100,
    "capability": 121
   },
   "repo": "https://huggingface.co/alibiserikbay/JevK5",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 37.351269273786905,
    "B": 33.15853199438306,
    "C": 25.938381440129795
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18882605116814374,
    "p50_s_raw": 0.019413025584071875,
    "p95_s_adjusted": 0.21931404622737313,
    "p95_s_raw": 0.03465702311368657
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.78876192812696,
      89.6881064689264
     ],
     "I": [
      32.336664057232596,
      43.498424875066746
     ],
     "capability": [
      59.02389559614775,
      65.70541220096806
     ],
     "composite": {
      "A": [
       24.235506619160738,
       49.5798377283499
      ],
      "B": [
       20.90293459315668,
       45.0236262225029
      ],
      "C": [
       16.83021292997273,
       34.43044286690966
      ]
     },
     "seed": 48
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.80312361577623,
     "capability": 64.16756807976074,
     "intelligence": 38.53201254374527
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 88.84490011579697,
      "cc": 60.099656129515054,
      "cc_by_type": {
       "choice": 60.099656129515054
      },
      "cc_tiered": 59.70206119042432,
      "cc_tiered_open": 60.58350447013268,
      "cc_tiered_sealed": 58.82061791071595,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.099656129515054
     },
     "noul": {
      "accuracy": 0.6213333333333333,
      "calibration": 89.24268928395378,
      "cc": 24.26666666666666,
      "cc_by_type": {
       "noul": 24.26666666666666
      },
      "cc_tiered": 21.681048401895232,
      "cc_tiered_open": 20.41614906832298,
      "cc_tiered_sealed": 22.94594773546748,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.26666666666666
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 91.32178144757796,
      "cc": 35.814600564516994,
      "cc_by_type": {
       "score": 35.814600564516994
      },
      "cc_tiered": 34.21292803891628,
      "cc_tiered_open": 34.00121908593804,
      "cc_tiered_sealed": 34.42463699189452,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.814600564516994
     }
    },
    "raw_on_A": {
     "calibration": 86.97080976439989,
     "capability": 64.96464026185238,
     "intelligence": 42.95847075930487
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 33.95922492821152,
    "Emphasis on Accuracy 60:20:20": 28.451132423327735,
    "Emphasis on Cost 20:20:60": 35.27745274054773,
    "Emphasis on Speed 20:60:20": 40.247060430803174,
    "Intelligence only": 22.883638070578947
   }
  },
  {
   "adapter_id": "images/metask-jev-4b/spec.json:metask_v15.MetaskJevWarm",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Wayfind (metask-ai)",
   "axes": {
    "calibration": 84.41575352936586,
    "cost": 57.7311327584882,
    "intelligence": 39.166454612935595,
    "speed": 90.45494495649334
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06577096449867435,
      "ece_score": 86.84580710026512,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 86.84580710026512,
      "score_before_time_limit": 86.84580710026512,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2506738544474394,
      "brier": 0.14719486396931378,
      "ece": 0.09558018432686369,
      "n": 371,
      "score": 80.88396313462727,
      "score_before_time_limit": 80.88396313462727,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.6723125564588976,
      "n": 369,
      "nrps": 0.11633460342816504,
      "score": 85.51749035320519,
      "score_before_time_limit": 85.51749035320519,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.08665779475386558
     }
    },
    "score": 84.41575352936586
   },
   "capability": 61.79110407115073,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     24.34423029596555,
     47.891744223948386
    ],
    "B": [
     21.360979157386925,
     43.89276263482132
    ],
    "C": [
     16.905715483309407,
     33.25815571107527
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025642610837438424
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-4b",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 41.89224435585361,
    "I_sealed": 36.440664870017585,
    "base": 39.166454612935595,
    "excess": 2.878191843397784,
    "gap": 5.451579485836028,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 60.50866528870256,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 52.815350739226176,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 37.846790890269155,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 27.321276888589132,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.68821311926259,
       "hard": 13.269014128057655,
       "judge": 46.73482367537818,
       "standard": 21.622014114131783
      }
     },
     "sealed|choice": {
      "cc": 56.24961082249847,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 73.3944276106718,
       "hard": 48.076923076923066,
       "judge": 61.824953445065184,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 22.8393257860911,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 2.9126213592232997,
       "judge": 23.684210526315795,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 30.23305800146319,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.21133272720072,
       "hard": 17.595550281640328,
       "judge": 37.810328121508064,
       "standard": 34.153030898172844
      }
     }
    }
   },
   "jevbench_score": 37.32666127648081,
   "key": "metask-jev-4b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8776978417266187,
    "decisive_rate": 0.7493261455525606,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.41575352936586,
     "cost": 57.7311327584882,
     "intelligence": 39.166454612935595,
     "speed": 90.78239008886905
    },
    "board_release": "v1.6.43",
    "capability": 61.79110407115073,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 53,
    "capability_rank": 78,
    "composite_ci95": {
     "A": [
      24.35782606760525,
      47.92198520846029
     ],
     "B": [
      21.36935197657226,
      43.913080458446345
     ],
     "C": [
      16.915156991392532,
      33.27915639476408
     ]
    },
    "jevbench_score": 37.34931073616377,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 57,
    "rank": 87,
    "ranks": {
     "A": 87,
     "B": 88,
     "C": 88,
     "capability": 122
    },
    "scores": {
     "A": 37.34931073616377,
     "B": 33.62317034896201,
     "C": 25.937021344558172
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25794456995790827,
     "p50_s_raw": 0.05397228497895412,
     "p95_s_adjusted": 0.3237685059779324,
     "p95_s_raw": 0.08688425298896618
    },
    "views": {
     "Balanced 33:33:33": 34.17198402783528,
     "Emphasis on Accuracy 60:20:20": 29.237864520407634,
     "Emphasis on Cost 20:20:60": 34.662048899004674,
     "Emphasis on Speed 20:60:20": 40.422015539591015,
     "Intelligence only": 24.032711498388675
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 99,
   "ranked": true,
   "ranks": {
    "A": 99,
    "B": 100,
    "C": 101,
    "capability": 138
   },
   "repo": "https://github.com/metask-ai/metask-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 37.32666127648081,
    "B": 33.608483329478155,
    "C": 25.92129255311167
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2660654459963553,
    "p50_s_raw": 0.05803272299817763,
    "p95_s_adjusted": 0.3384675424604211,
    "p95_s_raw": 0.09423377123021055
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-4B merged r16 LoRA, candidate-logit readout",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.04800226228174,
      85.90669927321129
     ],
     "I": [
      32.96914634416673,
      43.538341958330456
     ],
     "capability": [
      56.20112405588935,
      63.92462857658634
     ],
     "composite": {
      "A": [
       24.34423029596555,
       47.891744223948386
      ],
      "B": [
       21.360979157386925,
       43.89276263482132
      ],
      "C": [
       16.905715483309407,
       33.25815571107527
      ]
     },
     "seed": 66
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.41575352936586,
     "capability": 61.79110407115073,
     "intelligence": 39.166454612935595
    },
    "per_type": {
     "choice": {
      "accuracy": 0.668,
      "calibration": 86.84580710026512,
      "cc": 57.541941778842954,
      "cc_by_type": {
       "choice": 57.541941778842954
      },
      "cc_tiered": 58.379138055600514,
      "cc_tiered_open": 60.50866528870256,
      "cc_tiered_sealed": 56.24961082249847,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.541941778842954
     },
     "noul": {
      "accuracy": 0.6506666666666666,
      "calibration": 80.88396313462727,
      "cc": 30.133333333333322,
      "cc_by_type": {
       "noul": 30.133333333333322
      },
      "cc_tiered": 30.34305833818013,
      "cc_tiered_open": 37.846790890269155,
      "cc_tiered_sealed": 22.8393257860911,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 30.133333333333322
     },
     "score": {
      "accuracy": 0.48,
      "calibration": 85.51749035320519,
      "cc": 29.198392850325618,
      "cc_by_type": {
       "score": 29.198392850325618
      },
      "cc_tiered": 28.77716744502616,
      "cc_tiered_open": 27.321276888589132,
      "cc_tiered_sealed": 30.23305800146319,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.198392850325618
     }
    },
    "raw_on_A": {
     "calibration": 83.18850868556241,
     "capability": 62.528725908581364,
     "intelligence": 41.86894313160032
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 34.146707705957056,
    "Emphasis on Accuracy 60:20:20": 29.226758143927484,
    "Emphasis on Cost 20:20:60": 34.646440478782935,
    "Emphasis on Speed 20:60:20": 40.35840642875074,
    "Intelligence only": 24.032711498388675
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "krishna765",
   "axes": {
    "calibration": 88.59116686443139,
    "cost": 63.46145558281372,
    "intelligence": 38.62465958342338,
    "speed": 89.8569997500675
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06995357456541221,
      "ece_score": 86.00928508691756,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 86.00928508691756,
      "score_before_time_limit": 86.00928508691756,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3207547169811321,
      "brier": 0.11609579904406445,
      "ece": 0.05066343452944609,
      "n": 371,
      "score": 89.86731309411078,
      "score_before_time_limit": 89.86731309411078,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.47515808491418265,
      "n": 369,
      "nrps": 0.10083978781514244,
      "score": 89.89690241226586,
      "score_before_time_limit": 89.89690241226586,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.050611081969770266
     }
    },
    "score": 88.59116686443139
   },
   "capability": 63.60791322392738,
   "composite_ci95": {
    "A": [
     23.961513626426882,
     50.372221205412664
    ],
    "B": [
     20.748483970550282,
     45.81939026670353
    ],
    "C": [
     16.639940018352,
     34.98070917042547
    ]
   },
   "cost": {
    "basis": "ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra Qwen/Qwen3.5-4B list price $0.03/M input, $0.15/M output (base of JevK5, the adapter's base checkpoint; not on OpenRouter; frozen 25 Sep snapshot per I-3, same reference as reflex-4b, jobe, typecastlm and Quyet) x the server's own measured usage.input_tokens; logit readout, 0 output tokens; FP8 weight-only on 1x RTX PRO 6000 96GB (Lium; v1.6 GPU-class deviation from source H100), deck-4b 1.0.0 + jevk5 f944fe37; estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0162645
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01651777251184834
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "deck-4B v1.0 (krishna765, JevK5 + LoRA, FP8 weight-only)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 38.79947935146771,
    "I_sealed": 38.44983981537905,
    "base": 38.62465958342338,
    "excess": -2.2237481063495843,
    "gap": 0.3496395360886595,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 58.68740862543909,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 60.36489462094999,
       "judge": 64.40342781806196,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 25.017253278122844,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 0.0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 32.6937761508412,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.877671536622636,
       "hard": 34.178023262130495,
       "judge": 35.59214965227232,
       "standard": 15.785773983225237
      }
     },
     "sealed|choice": {
      "cc": 59.157265682164855,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 56.22171945701356,
       "judge": 59.031657355679705,
       "standard": 60.09360758714129
      }
     },
     "sealed|noul": {
      "cc": 21.704287246216186,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": 0.9708737864077666,
       "judge": 31.578947368421062,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 34.48796651775611,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.00896406653321,
       "hard": 26.22895515401159,
       "judge": 35.59993017640196,
       "standard": 33.57754498288781
      }
     }
    }
   },
   "jevbench_score": 37.259120408179,
   "key": "deck-4b-v1-0",
   "last_measured_on": "2026-10-05",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9325396825396826,
    "decisive_rate": 0.6792452830188679,
    "supported": true,
    "valid": 371
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.59116686443139,
     "cost": 63.46145558281372,
     "intelligence": 38.62465958342338,
     "speed": 89.93848373337222
    },
    "board_release": "v1.6.43",
    "capability": 63.60791322392738,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 43,
    "capability_rank": 67,
    "composite_ci95": {
     "A": [
      23.964977734953006,
      50.38055037709117
     ],
     "B": [
      20.750561783148846,
      45.824903274791566
     ],
     "C": [
      16.642345649272922,
      34.98649331742443
     ]
    },
    "jevbench_score": 37.26498528662829,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 58,
    "rank": 88,
    "ranks": {
     "A": 88,
     "B": 89,
     "C": 89,
     "capability": 109
    },
    "scores": {
     "A": 37.26498528662829,
     "B": 33.17299510735835,
     "C": 25.87846200460298
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.30241732355207207,
     "p50_s_raw": 0.07620866177603602,
     "p95_s_adjusted": 0.3353860241360962,
     "p95_s_raw": 0.09269301206804811
    },
    "views": {
     "Balanced 33:33:33": 33.95960169128873,
     "Emphasis on Accuracy 60:20:20": 28.553229571667234,
     "Emphasis on Cost 20:20:60": 35.46476300764428,
     "Emphasis on Speed 20:60:20": 39.807414011248,
     "Intelligence only": 23.04910072446208
    }
   },
   "provenance": {
    "carried_metadata_from": null,
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 100,
   "ranked": true,
   "ranks": {
    "A": 100,
    "B": 101,
    "C": 102,
    "capability": 127
   },
   "repo": "https://github.com/krishna-gogineni-765/deck",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 37.259120408179,
    "B": 33.169276878315856,
    "C": 25.87438917234653
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.30445735771209004,
    "p50_s_raw": 0.07722867885604501,
    "p95_s_adjusted": 0.33944823201745744,
    "p95_s_raw": 0.09472411600872874
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.64482190603961,
      89.1506890221124
     ],
     "I": [
      32.29690931230169,
      43.8177901770495
     ],
     "capability": [
      58.30292793082468,
      65.68855559803174
     ],
     "composite": {
      "A": [
       23.961513626426882,
       50.372221205412664
      ],
      "B": [
       20.748483970550282,
       45.81939026670353
      ],
      "C": [
       16.639940018352,
       34.98070917042547
      ]
     },
     "seed": 33
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.59116686443139,
     "capability": 63.60791322392738,
     "intelligence": 38.62465958342338
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 86.00928508691756,
      "cc": 60.099656129515054,
      "cc_by_type": {
       "choice": 60.099656129515054
      },
      "cc_tiered": 58.922337153801976,
      "cc_tiered_open": 58.68740862543909,
      "cc_tiered_sealed": 59.157265682164855,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.099656129515054
     },
     "noul": {
      "accuracy": 0.6266666666666667,
      "calibration": 89.86731309411078,
      "cc": 25.333333333333343,
      "cc_by_type": {
       "noul": 25.333333333333343
      },
      "cc_tiered": 23.360770262169517,
      "cc_tiered_open": 25.017253278122844,
      "cc_tiered_sealed": 21.704287246216186,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.333333333333343
     },
     "score": {
      "accuracy": 0.5546666666666666,
      "calibration": 89.89690241226586,
      "cc": 35.52454801522435,
      "cc_by_type": {
       "score": 35.52454801522435
      },
      "cc_tiered": 33.590871334298654,
      "cc_tiered_open": 32.6937761508412,
      "cc_tiered_sealed": 34.48796651775611,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.52454801522435
     }
    },
    "raw_on_A": {
     "calibration": 89.0005990160775,
     "capability": 66.05004010202693,
     "intelligence": 43.09948118797635
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 33.92104982455805,
    "Emphasis on Accuracy 60:20:20": 28.536867950514182,
    "Emphasis on Cost 20:20:60": 35.39762374246159,
    "Emphasis on Speed 20:60:20": 39.76493034255667,
    "Intelligence only": 23.04910072446208
   }
  },
  {
   "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/clef-flash.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 87.5533640278553,
     "cost": 48.17236064887202,
     "intelligence": 53.08161490373422,
     "speed": 83.63107135213085
    },
    "label": "Workers AI price scenario (latency unmeasured)",
    "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.",
    "pricing_only": true,
    "usd_per_1000": 0.053405745073891626
   },
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Cloudflare",
   "axes": {
    "calibration": 85.46479352633332,
    "cost": 46.79963593205177,
    "intelligence": 42.18368888957246,
    "speed": 88.14167964175255
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1119512500000001,
      "ece_score": 77.60974999999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 77.60974999999998,
      "score_before_time_limit": 77.60974999999998,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.31733333333333336,
      "brier": 0.11674051933333338,
      "ece": 0.04965040000000006,
      "n": 375,
      "score": 90.06992,
      "score_before_time_limit": 90.06992,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9333333333333333,
      "interval90_width": 0.460888888888889,
      "n": 375,
      "nrps": 0.09493298842,
      "score": 88.71471057900001,
      "score_before_time_limit": 88.71471057900001,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06538639999999996
     }
    },
    "score": 85.46479352633332
   },
   "capability": 63.824241207952895,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     26.627544279671433,
     45.19975127419028
    ],
    "B": [
     24.1832535120261,
     42.555740475244534
    ],
    "C": [
     18.49135019421627,
     31.388716162632143
    ]
   },
   "cost": {
    "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; self-hosted estimate, not measured hosted billing",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05933971674876847
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Clef-Flash (Cloudflare, Qwen3.5-9B post-train with a joint schema head, multimodal, measured on text)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 45.25550449770323,
    "I_sealed": 39.11187328144169,
    "base": 42.18368888957246,
    "excess": 3.5702435738232956,
    "gap": 6.143631216261539,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 62.93462232292448,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 56.59012268008808,
       "judge": 68.35860250494396,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 30.42788129744652,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": 0.0,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 42.4040098727387,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.5230303030303,
       "hard": 37.280925925925935,
       "judge": 43.06322033898306,
       "standard": 52.101851851851855
      }
     },
     "sealed|choice": {
      "cc": 54.80150596378907,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 45.53167420814478,
       "judge": 53.44506517690876,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 20.824374761599753,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52.0,
       "hard": -10.679611650485432,
       "judge": 31.578947368421062,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 41.70973911893625,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 54.71716312056738,
       "hard": 35.80301470588234,
       "judge": 41.72143712574849,
       "standard": 47.00192893401015
      }
     }
    }
   },
   "jevbench_score": 36.61691640066131,
   "key": "clef-flash",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Cloudflare/clef-flash@17f0b0ad64efb65d273590632833508766b2aae6",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.93359375,
    "decisive_rate": 0.6826666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.46479352633332,
     "cost": 46.79963593205177,
     "intelligence": 42.18368888957246,
     "speed": 88.4044196893156
    },
    "board_release": "v1.6.43",
    "capability": 63.824241207952895,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 41,
    "capability_rank": 64,
    "composite_ci95": {
     "A": [
      26.640037750072654,
      45.22280470107504
     ],
     "B": [
      24.191496498186186,
      42.57204972938278
     ],
     "C": [
      18.500026215328234,
      31.40472548685768
     ]
    },
    "jevbench_score": 36.635050479655426,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 60,
    "rank": 90,
    "ranks": {
     "A": 90,
     "B": 86,
     "C": 91,
     "capability": 106
    },
    "scores": {
     "A": 36.635050479655426,
     "B": 33.967249202727025,
     "C": 25.44100727753849
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3556603590026498,
     "p50_s_raw": 0.10283017950132489,
     "p95_s_adjusted": 0.405996760725975,
     "p95_s_raw": 0.12799838036298752
    },
    "views": {
     "Balanced 33:33:33": 33.178006784894365,
     "Emphasis on Accuracy 60:20:20": 30.03860559786673,
     "Emphasis on Cost 20:20:60": 31.455768602881705,
     "Emphasis on Speed 20:60:20": 39.463018165391404,
     "Intelligence only": 26.30501767147255
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 102,
   "ranked": true,
   "ranks": {
    "A": 102,
    "B": 98,
    "C": 104,
    "capability": 124
   },
   "repo": "https://huggingface.co/Cloudflare/clef-flash",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 36.61691640066131,
    "B": 33.95477628153724,
    "C": 25.428414167125908
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3573765229433775,
    "p50_s_raw": 0.10368826147168875,
    "p95_s_adjusted": 0.42924576271325354,
    "p95_s_raw": 0.13962288135662676
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.4975513933287,
      86.84610174167315
     ],
     "I": [
      36.95680839342918,
      46.203587775711
     ],
     "capability": [
      59.38252934298857,
      65.53439425442862
     ],
     "composite": {
      "A": [
       26.627544279671433,
       45.19975127419028
      ],
      "B": [
       24.1832535120261,
       42.555740475244534
      ],
      "C": [
       18.49135019421627,
       31.388716162632143
      ]
     },
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.46479352633332,
     "capability": 63.824241207952895,
     "intelligence": 42.18368888957246
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6666666666666666,
      "calibration": 77.60974999999998,
      "cc": 57.37142748879814,
      "cc_by_type": {
       "choice": 57.37142748879814
      },
      "cc_tiered": 58.868064143356776,
      "cc_tiered_open": 62.93462232292448,
      "cc_tiered_sealed": 54.80150596378907,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.37142748879814
     },
     "noul": {
      "accuracy": 0.6373333333333333,
      "calibration": 90.06992,
      "cc": 27.46666666666666,
      "cc_by_type": {
       "noul": 27.46666666666666
      },
      "cc_tiered": 25.626128029523137,
      "cc_tiered_open": 30.42788129744652,
      "cc_tiered_sealed": 20.824374761599753,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 27.46666666666666
     },
     "score": {
      "accuracy": 0.6026666666666667,
      "calibration": 88.71471057900001,
      "cc": 42.01200877192982,
      "cc_by_type": {
       "score": 42.01200877192982
      },
      "cc_tiered": 42.05687449583748,
      "cc_tiered_open": 42.4040098727387,
      "cc_tiered_sealed": 41.70973911893625,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.01200877192982
     }
    },
    "raw_on_A": {
     "calibration": 82.15484704393162,
     "capability": 63.82820945816345,
     "intelligence": 45.501571872395274
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 33.15817797600965,
    "Emphasis on Accuracy 60:20:20": 30.028850639820536,
    "Emphasis on Cost 20:20:60": 31.44507165653421,
    "Emphasis on Speed 20:60:20": 39.41255748561944,
    "Intelligence only": 26.30501767147255
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Eval Engine / Chromia",
   "axes": {
    "calibration": 80.25524974800695,
    "cost": 65.87776871337948,
    "intelligence": 37.93489123230218,
    "speed": 90.61339377257747
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11555442846600955,
      "ece_score": 76.88911430679809,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 76.88911430679809,
      "score_before_time_limit": 76.88911430679809,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.27762803234501343,
      "brier": 0.14312850498011134,
      "ece": 0.09433112828523382,
      "n": 371,
      "score": 81.13377434295323,
      "score_before_time_limit": 81.13377434295323,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.907859078590786,
      "interval90_width": 0.43202348690153575,
      "n": 369,
      "nrps": 0.11270639836290829,
      "score": 82.74286059426956,
      "score_before_time_limit": 82.74286059426956,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.11621819487585025
     }
    },
    "score": 80.25524974800695
   },
   "capability": 59.09507049015456,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     21.738459277283113,
     47.30478605027541
    ],
    "B": [
     18.69312914880245,
     42.948050859217794
    ],
    "C": [
     15.096152275891052,
     32.85054586824682
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off, already marked deprecated at that cutoff (replaced by Qwen/Qwen3.5-9B); re-score if it changes (I-3). Frozen market reference deepinfra:Qwen/Qwen3.5-4B.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013721693349753695
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision-4B (Eval Engine / Chromia)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P88 on one evaluator-owned Lium H100. Current staged log verifies evalengine/decision-4b-gguf@08859d1d8307f7b7ea452549e303b7805c35e350; stored GGUF SHA256 f2519bcdc945291974c81a37ff92417b22df27154c9df2564420d08576b907ce. Serial offline loopback-only inference, standard selfhosted x2+0.15 s latency adjustment. Carried published estimate retained.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 39.60182572574049,
    "I_sealed": 36.267956738863866,
    "base": 37.93489123230218,
    "excess": 0.76048134443838,
    "gap": 3.333868986876624,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 65.98784234179594,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 50.92796476879523,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 23.933747412008287,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 21.739130434782616,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 28.883887423417242,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.4752733983845,
       "hard": 37.0901498391321,
       "judge": 25.649863025499585,
       "standard": 7.526706201380384
      }
     },
     "sealed|choice": {
      "cc": 58.09043969346601,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 47.05882352941176,
       "judge": 60.89385474860336,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 19.731441484882723,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": -12.621359223300976,
       "judge": 28.947368421052634,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 30.981989038242872,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.61623338387761,
       "hard": 20.121162500023082,
       "judge": 32.27985452492822,
       "standard": 33.439721711837066
      }
     }
    }
   },
   "jevbench_score": 35.402914218057774,
   "key": "evalengine-decision-4b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "evalengine/decision-4b-gguf@08859d1d8307f7b7ea452549e303b7805c35e350",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8843283582089553,
    "decisive_rate": 0.7223719676549866,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 80.25524974800695,
     "cost": 65.87776871337948,
     "intelligence": 37.93489123230218,
     "speed": 91.06043780349009
    },
    "board_release": "v1.6.43",
    "capability": 59.09507049015456,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 63,
    "capability_rank": 88,
    "composite_ci95": {
     "A": [
      21.75506402712618,
      47.346177294331085
     ],
     "B": [
      18.702949452781887,
      42.97523990261764
     ],
     "C": [
      15.107683352170962,
      32.879289787729924
     ]
    },
    "jevbench_score": 35.43243109909669,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 61,
    "rank": 91,
    "ranks": {
     "A": 91,
     "B": 92,
     "C": 92,
     "capability": 137
    },
    "scores": {
     "A": 35.43243109909669,
     "B": 31.508666798854446,
     "C": 24.60585492992826
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24496516967192292,
     "p50_s_raw": 0.04748258483596146,
     "p95_s_adjusted": 0.3197804291266948,
     "p95_s_raw": 0.08489021456334739
    },
    "views": {
     "Balanced 33:33:33": 32.87874158865029,
     "Emphasis on Accuracy 60:20:20": 27.346985238704878,
     "Emphasis on Cost 20:20:60": 34.72560202914835,
     "Emphasis on Speed 20:60:20": 38.639773156510174,
     "Intelligence only": 21.836172722245216
    }
   },
   "provenance": {
    "carried_metadata_from": "Current inventory + r20 source metadata; unresolved runtime join",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 103,
   "ranked": true,
   "ranks": {
    "A": 103,
    "B": 105,
    "C": 105,
    "capability": 155
   },
   "repo": "https://huggingface.co/evalengine/decision-4b-gguf",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 35.402914218057774,
    "B": 31.48998911474531,
    "C": 24.585357095873455
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26147820902988317,
    "p50_s_raw": 0.05573910451494157,
    "p95_s_adjusted": 0.33206654312089084,
    "p95_s_raw": 0.09103327156044543
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.03864735316435,
      82.85641931336254
     ],
     "I": [
      31.11663059494919,
      42.88120054683658
     ],
     "capability": [
      54.031339191317976,
      62.320753924360694
     ],
     "composite": {
      "A": [
       21.738459277283113,
       47.30478605027541
      ],
      "B": [
       18.69312914880245,
       42.948050859217794
      ],
      "C": [
       15.096152275891052,
       32.85054586824682
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.25524974800695,
     "capability": 59.09507049015456,
     "intelligence": 37.93489123230218
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 76.88911430679809,
      "cc": 59.92914183947025,
      "cc_by_type": {
       "choice": 59.92914183947025
      },
      "cc_tiered": 62.039141017630975,
      "cc_tiered_open": 65.98784234179594,
      "cc_tiered_sealed": 58.09043969346601,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.92914183947025
     },
     "noul": {
      "accuracy": 0.632,
      "calibration": 81.13377434295323,
      "cc": 26.400000000000002,
      "cc_by_type": {
       "noul": 26.400000000000002
      },
      "cc_tiered": 21.832594448445505,
      "cc_tiered_open": 23.933747412008287,
      "cc_tiered_sealed": 19.731441484882723,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 26.400000000000002
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 82.74286059426956,
      "cc": 32.24707656205765,
      "cc_by_type": {
       "score": 32.24707656205765
      },
      "cc_tiered": 29.932938230830057,
      "cc_tiered_open": 28.883887423417242,
      "cc_tiered_sealed": 30.981989038242872,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.24707656205765
     }
    },
    "raw_on_A": {
     "calibration": 78.19048224326764,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 32.84486092944197,
    "Emphasis on Accuracy 60:20:20": 27.332914532747036,
    "Emphasis on Cost 20:20:60": 34.7029171612182,
    "Emphasis on Speed 20:60:20": 38.55564057136092,
    "Intelligence only": 21.836172722245216
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Icarus AI / cmxu",
   "axes": {
    "calibration": 88.12743013642095,
    "cost": 64.41461114142905,
    "intelligence": 37.57770128683752,
    "speed": 92.71898033409317
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08741578110266195,
      "ece_score": 82.5168437794676,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 82.5168437794676,
      "score_before_time_limit": 82.5168437794676,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.36657681940700804,
      "brier": 0.12068971845453581,
      "ece": 0.04093580710766913,
      "n": 371,
      "score": 91.81283857846617,
      "score_before_time_limit": 91.81283857846617,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9376693766937669,
      "interval90_width": 0.4548328816621501,
      "n": 369,
      "nrps": 0.10039054297386547,
      "score": 90.05260805132909,
      "score_before_time_limit": 90.05260805132909,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.049278647999776395
     }
    },
    "score": 88.12743013642095
   },
   "capability": 62.85256571162924,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     21.828170552163172,
     45.51972901414073
    ],
    "B": [
     18.714446379791095,
     41.10503731892094
    ],
    "C": [
     15.158451772335539,
     31.610922926486623
    ]
   },
   "cost": {
    "basis": "add-requests run 47 (v1.5 basis, unpublished) base-model price floor deepinfra:Qwen/Qwen3.5-4B",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015352518472906404
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "janus 4B (Icarus AI / cmxu, Qwen3.5-4B + LoRA r64 and pointer decision head)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000",
   "intelligence": {
    "I_open": 40.660351519488046,
    "I_sealed": 34.49505105418699,
    "base": 37.57770128683752,
    "excess": 3.5919128228628145,
    "gap": 6.165300465301058,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.67923497711049,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 23.289164941338854,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 47.82608695652173,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 37.01265464001479,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.99228809649041,
       "hard": 42.7176115493677,
       "judge": 46.3643071982838,
       "standard": 6.5854452556676435
      }
     },
     "sealed|choice": {
      "cc": 56.292971028212264,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 66.74303451333975,
       "hard": 49.60407239819003,
       "judge": 56.238361266294234,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 12.558893678884758,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": -14.563106796116509,
       "judge": 23.684210526315795,
       "standard": 32.3943661971831
      }
     },
     "sealed|score": {
      "cc": 34.633288455463955,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 59.65725589324873,
       "hard": 29.365017382365,
       "judge": 35.41609562746013,
       "standard": 31.483636124775206
      }
     }
    }
   },
   "jevbench_score": 35.15464029738947,
   "key": "janus-4b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "cmxu/janus-4b@fc86a8fbaa99de8dbac5aee46aaa56d1a853b876",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9446808510638298,
    "decisive_rate": 0.633423180592992,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.12743013642095,
     "cost": 64.41461114142905,
     "intelligence": 37.57770128683752,
     "speed": 93.17471817821269
    },
    "board_release": "v1.6.43",
    "capability": 62.85256571162924,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 46,
    "capability_rank": 71,
    "composite_ci95": {
     "A": [
      21.844531574541936,
      45.558507177419784
     ],
     "B": [
      18.72406374533524,
      41.13032827742159
     ],
     "C": [
      15.169813593431902,
      31.63785220654152
     ]
    },
    "jevbench_score": 35.18351981268845,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 63,
    "rank": 93,
    "ranks": {
     "A": 93,
     "B": 95,
     "C": 94,
     "capability": 113
    },
    "scores": {
     "A": 35.18351981268845,
     "B": 31.093845400105046,
     "C": 24.432999869922536
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2097848016768694,
     "p50_s_raw": 0.029892400838434696,
     "p95_s_adjusted": 0.22948492439463736,
     "p95_s_raw": 0.039742462197318675
    },
    "views": {
     "Balanced 33:33:33": 32.05124045782333,
     "Emphasis on Accuracy 60:20:20": 26.620100423360302,
     "Emphasis on Cost 20:20:60": 33.65415776578004,
     "Emphasis on Speed 20:60:20": 37.99318109571395,
     "Intelligence only": 21.225142796235538
    }
   },
   "provenance": {
    "carried_metadata_from": "run 47 author metadata + current P83 staged pin/measurement receipt",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 105,
   "ranked": true,
   "ranks": {
    "A": 105,
    "B": 108,
    "C": 107,
    "capability": 130
   },
   "repo": "https://huggingface.co/cmxu/janus-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 35.15464029738947,
    "B": 31.07579633776246,
    "C": 24.41294465096491
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2128553358837962,
    "p50_s_raw": 0.03142766794189811,
    "p95_s_adjusted": 0.2511987216770648,
    "p95_s_raw": 0.05059936083853242
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.8809685848649,
      88.36140314243593
     ],
     "I": [
      31.000587963045195,
      41.97868333470401
     ],
     "capability": [
      57.23309571773392,
      64.21484535464957
     ],
     "composite": {
      "A": [
       21.828170552163172,
       45.51972901414073
      ],
      "B": [
       18.714446379791095,
       41.10503731892094
      ],
      "C": [
       15.158451772335539,
       31.610922926486623
      ]
     },
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.12743013642095,
     "capability": 62.85256571162924,
     "intelligence": 37.57770128683752
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6733333333333333,
      "calibration": 82.5168437794676,
      "cc": 58.22399893902218,
      "cc_by_type": {
       "choice": 58.22399893902218
      },
      "cc_tiered": 58.98610300266138,
      "cc_tiered_open": 61.67923497711049,
      "cc_tiered_sealed": 56.292971028212264,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.22399893902218
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 91.81283857846617,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 17.924029310111806,
      "cc_tiered_open": 23.289164941338854,
      "cc_tiered_sealed": 12.558893678884758,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 90.05260805132909,
      "cc": 36.47974838634844,
      "cc_by_type": {
       "score": 36.47974838634844
      },
      "cc_tiered": 35.82297154773937,
      "cc_tiered_open": 37.01265464001479,
      "cc_tiered_sealed": 34.633288455463955,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.47974838634844
     }
    },
    "raw_on_A": {
     "calibration": 84.32063965516178,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 32.01929101807324,
    "Emphasis on Accuracy 60:20:20": 26.60687037514617,
    "Emphasis on Cost 20:20:60": 33.633014967297186,
    "Emphasis on Speed 20:60:20": 37.91246375151397,
    "Intelligence only": 21.225142796235538
   }
  },
  {
   "adapter_id": "images/qwen35-9b-jev-data-mix-v2/spec.json:qwen35_nimble_lora.Qwen35NimbleLoraAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "jsaurabh",
   "axes": {
    "calibration": 79.78895435560509,
    "cost": 45.690218065856605,
    "intelligence": 41.94184606895763,
    "speed": 85.22302455389745
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14238786731364284,
      "ece_score": 71.52242653727143,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 71.52242653727143,
      "score_before_time_limit": 71.52242653727143,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2911051212938005,
      "brier": 0.12127289774594222,
      "ece": 0.07874545275117417,
      "n": 371,
      "score": 84.25090944976516,
      "score_before_time_limit": 84.25090944976516,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8888888888888888,
      "interval90_width": 0.31820234869015335,
      "n": 369,
      "nrps": 0.0937862889000391,
      "score": 83.59352707977865,
      "score_before_time_limit": 83.59352707977865,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.11717158475219394
     }
    },
    "score": 79.78895435560509
   },
   "capability": 60.86540021228136,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     22.85464148652355,
     41.72926390773847
    ],
    "B": [
     20.748491326322572,
     39.44219459722902
    ],
    "C": [
     15.871278810085796,
     28.97865549148505
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06461391625615764
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Qwen3.5-9B Jev-like data-mix v2",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 45.69949425428299,
    "I_sealed": 38.184197883632265,
    "base": 41.94184606895763,
    "excess": 4.94190872821248,
    "gap": 7.515296370650724,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 56.305297450788785,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 49.040578798364265,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 33.4120082815735,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 47.38117703048668,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.73251602617265,
       "hard": 54.043486375775075,
       "judge": 35.50551913484291,
       "standard": 50.19437568553258
      }
     },
     "sealed|choice": {
      "cc": 54.756843153610475,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 46.04072398190045,
       "judge": 51.58286778398511,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 20.448763917177054,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52.0,
       "hard": -4.854368932038833,
       "judge": 26.315789473684205,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 39.346986580109274,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.00077177598622,
       "hard": 33.3124721590493,
       "judge": 38.59270672300753,
       "standard": 37.720542609943344
      }
     }
    }
   },
   "jevbench_score": 33.57747577458131,
   "key": "qwen35-9b-jev-data-mix-v2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (adapter code and weights; Qwen3.5-9B base is Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.908745247148289,
    "decisive_rate": 0.7088948787061995,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.78895435560509,
     "cost": 45.690218065856605,
     "intelligence": 41.94184606895763,
     "speed": 85.40793837616167
    },
    "board_release": "v1.6.43",
    "capability": 60.86540021228136,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      22.862450507726937,
      41.74491188365231
     ],
     "B": [
      20.753639028685242,
      39.453425031751074
     ],
     "C": [
      15.87670174147704,
      28.989522141425212
     ]
    },
    "jevbench_score": 33.58966697347588,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 64,
    "rank": 94,
    "ranks": {
     "A": 94,
     "B": 93,
     "C": 96,
     "capability": 129
    },
    "scores": {
     "A": 33.58966697347588,
     "B": 31.31610274190126,
     "C": 23.326157620469353
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4865717590786517,
     "p50_s_raw": 0.16828587953932583,
     "p95_s_adjusted": 0.5916423641145229,
     "p95_s_raw": 0.22082118205726145
    },
    "views": {
     "Balanced 33:33:33": 30.68928023743452,
     "Emphasis on Accuracy 60:20:20": 27.947003437168036,
     "Emphasis on Cost 20:20:60": 29.02722060908932,
     "Emphasis on Speed 20:60:20": 36.335136837664905,
     "Intelligence only": 24.643879144700183
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 106,
   "ranked": true,
   "ranks": {
    "A": 106,
    "B": 106,
    "C": 109,
    "capability": 147
   },
   "repo": "https://huggingface.co/jsaurabh/qwen3.5-9b-jev-data-mix-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 33.57747577458131,
    "B": 31.3076246037687,
    "C": 23.317691510125904
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.49227979183197024,
    "p50_s_raw": 0.1711398959159851,
    "p95_s_adjusted": 0.6102187336888164,
    "p95_s_raw": 0.23010936684440822
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.15659713819453,
      82.50836404329671
     ],
     "I": [
      35.70342834084104,
      45.9106569165651
     ],
     "capability": [
      56.2798648487553,
      63.69018289459294
     ],
     "composite": {
      "A": [
       22.85464148652355,
       41.72926390773847
      ],
      "B": [
       20.748491326322572,
       39.44219459722902
      ],
      "C": [
       15.871278810085796,
       28.97865549148505
      ]
     },
     "seed": 92
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.78895435560509,
     "capability": 60.86540021228136,
     "intelligence": 41.94184606895763
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6533333333333333,
      "calibration": 71.52242653727143,
      "cc": 55.666284588350074,
      "cc_by_type": {
       "choice": 55.666284588350074
      },
      "cc_tiered": 55.53107030219963,
      "cc_tiered_open": 56.305297450788785,
      "cc_tiered_sealed": 54.756843153610475,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.666284588350074
     },
     "noul": {
      "accuracy": 0.6373333333333333,
      "calibration": 84.25090944976516,
      "cc": 27.46666666666666,
      "cc_by_type": {
       "noul": 27.46666666666666
      },
      "cc_tiered": 26.930386099375276,
      "cc_tiered_open": 33.4120082815735,
      "cc_tiered_sealed": 20.448763917177054,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 27.46666666666666
     },
     "score": {
      "accuracy": 0.5733333333333334,
      "calibration": 83.59352707977865,
      "cc": 42.24339448080683,
      "cc_by_type": {
       "score": 42.24339448080683
      },
      "cc_tiered": 43.36408180529798,
      "cc_tiered_open": 47.38117703048668,
      "cc_tiered_sealed": 39.346986580109274,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.24339448080683
     }
    },
    "raw_on_A": {
     "calibration": 80.51953554017331,
     "capability": 63.66543855723072,
     "intelligence": 46.811341574288136
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 30.675712333180403,
    "Emphasis on Accuracy 60:20:20": 27.940251191797522,
    "Emphasis on Cost 20:20:60": 29.019936363652732,
    "Emphasis on Speed 20:60:20": 36.30091933263128,
    "Intelligence only": 24.643879144700183
   }
  },
  {
   "adapter_id": "images/jevk5-v02/spec.json:jevk5_direct.JevK5DirectAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "allebee",
   "axes": {
    "calibration": 89.38710082819252,
    "cost": 63.07359395281681,
    "intelligence": 36.261123811123895,
    "speed": 91.22516918010956
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05722074836783767,
      "ece_score": 88.55585032643246,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.55585032643246,
      "score_before_time_limit": 88.55585032643246,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3800539083557951,
      "brier": 0.12953478188668033,
      "ece": 0.0506873602652232,
      "n": 371,
      "score": 89.86252794695537,
      "score_before_time_limit": 89.86252794695537,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.981029810298103,
      "interval90_width": 0.5517163504968384,
      "n": 369,
      "nrps": 0.098178280103309,
      "score": 89.74292421118973,
      "score_before_time_limit": 89.74292421118973,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.053481617836448225
     }
    },
    "score": 89.38710082819252
   },
   "capability": 62.824112319658205,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     21.320207406790786,
     41.4566516992432
    ],
    "B": [
     18.2591367748241,
     37.24188924663158
    ],
    "C": [
     14.805699588049155,
     28.78934145780778
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01701689039408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JevK5 v0.2.0",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 39.85943426682522,
    "I_sealed": 32.66281335542257,
    "base": 36.261123811123895,
    "excess": 4.623233268964404,
    "gap": 7.196620911402647,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 56.2584494703768,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 52.815350739226176,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 29.9399585921325,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 47.82608695652173,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 33.37989473796636,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.905656531340256,
       "hard": 27.673703730451283,
       "judge": 40.84579963079196,
       "standard": 20.83053851707115
      }
     },
     "sealed|choice": {
      "cc": 55.096468823579634,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 55.20361990950226,
       "judge": 51.58286778398511,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 6.636284338632712,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -14.563106796116509,
       "judge": 18.421052631578938,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 36.25568690405537,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.58927438023056,
       "hard": 27.57268337567902,
       "judge": 41.004484216055225,
       "standard": 32.331704254720705
      }
     }
    }
   },
   "jevbench_score": 32.07906231997491,
   "key": "jevk5-v02",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code/adapter); Apache-2.0 (Qwen3.5-4B base)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9347826086956522,
    "decisive_rate": 0.6199460916442049,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.38710082819252,
     "cost": 63.07359395281681,
     "intelligence": 36.261123811123895,
     "speed": 91.56570599843724
    },
    "board_release": "v1.6.43",
    "capability": 62.824112319658205,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 47,
    "capability_rank": 72,
    "composite_ci95": {
     "A": [
      21.332485805393638,
      41.48362112591029
     ],
     "B": [
      18.26634005992914,
      37.259117524916284
     ],
     "C": [
      14.814226253745582,
      28.808070226326592
     ]
    },
    "jevbench_score": 32.09901620353128,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 66,
    "rank": 97,
    "ranks": {
     "A": 97,
     "B": 99,
     "C": 99,
     "capability": 114
    },
    "scores": {
     "A": 32.09901620353128,
     "B": 28.24080105110956,
     "C": 22.290983474674498
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23078905596630647,
     "p50_s_raw": 0.04039452798315324,
     "p95_s_adjusted": 0.30214414960937575,
     "p95_s_raw": 0.07607207480468788
    },
    "views": {
     "Balanced 33:33:33": 29.02935442351159,
     "Emphasis on Accuracy 60:20:20": 24.01393367218798,
     "Emphasis on Cost 20:20:60": 30.556198079902455,
     "Emphasis on Speed 20:60:20": 34.51299399305244,
     "Intelligence only": 19.071452492870637
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 108,
   "ranked": true,
   "ranks": {
    "A": 108,
    "B": 113,
    "C": 111,
    "capability": 132
   },
   "repo": "https://github.com/allebee/jevk5",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 32.07906231997491,
    "B": 28.228442488528938,
    "C": 22.27712661109369
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23864530100254341,
    "p50_s_raw": 0.04432265050127171,
    "p95_s_adjusted": 0.31603137055644764,
    "p95_s_raw": 0.08301568527822382
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.43880788393268,
      90.09857764344801
     ],
     "I": [
      30.721600512376973,
      40.44080848333523
     ],
     "capability": [
      58.00843038572983,
      64.2177747361447
     ],
     "composite": {
      "A": [
       21.320207406790786,
       41.4566516992432
      ],
      "B": [
       18.2591367748241,
       37.24188924663158
      ],
      "C": [
       14.805699588049155,
       28.78934145780778
      ]
     },
     "seed": 49
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.38710082819252,
     "capability": 62.824112319658205,
     "intelligence": 36.261123811123895
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6546666666666666,
      "calibration": 88.55585032643246,
      "cc": 55.836798878394866,
      "cc_by_type": {
       "choice": 55.836798878394866
      },
      "cc_tiered": 55.67745914697822,
      "cc_tiered_open": 56.2584494703768,
      "cc_tiered_sealed": 55.096468823579634,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.836798878394866
     },
     "noul": {
      "accuracy": 0.5733333333333334,
      "calibration": 89.86252794695537,
      "cc": 14.666666666666671,
      "cc_by_type": {
       "noul": 14.666666666666671
      },
      "cc_tiered": 18.288121465382606,
      "cc_tiered_open": 29.9399585921325,
      "cc_tiered_sealed": 6.636284338632712,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 14.666666666666671
     },
     "score": {
      "accuracy": 0.5546666666666666,
      "calibration": 89.74292421118973,
      "cc": 36.384921485746844,
      "cc_by_type": {
       "score": 36.384921485746844
      },
      "cc_tiered": 34.81779082101087,
      "cc_tiered_open": 33.37989473796636,
      "cc_tiered_sealed": 36.25568690405537,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.384921485746844
     }
    },
    "raw_on_A": {
     "calibration": 85.60375573005521,
     "capability": 62.620633576812004,
     "intelligence": 39.637511423568796
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 29.007597277916414,
    "Emphasis on Accuracy 60:20:20": 24.004997140522484,
    "Emphasis on Cost 20:20:60": 30.541730463341658,
    "Emphasis on Speed 20:60:20": 34.45768522015898,
    "Intelligence only": 19.071452492870637
   }
  },
  {
   "adapter_id": "images/jevone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Juspay",
   "axes": {
    "calibration": 91.76581828544445,
    "cost": 39.84212731591276,
    "intelligence": 45.955077428728934,
    "speed": 89.58480866139566
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.034452812499999874,
      "ece_score": 93.10943750000003,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 93.10943750000003,
      "score_before_time_limit": 93.10943750000003,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.376,
      "brier": 0.10284091864000006,
      "ece": 0.03908880000000003,
      "n": 375,
      "score": 92.18224,
      "score_before_time_limit": 92.18224,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9706666666666667,
      "interval90_width": 0.3935555555555556,
      "n": 375,
      "nrps": 0.08401831953999996,
      "score": 90.00577735633334,
      "score_before_time_limit": 90.00577735633334,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05793306666666674
     }
    },
    "score": 91.76581828544445
   },
   "capability": 68.8604478570867,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     23.176346265212082,
     37.82109096021405
    ],
    "B": [
     21.595080696244864,
     36.549542985952485
    ],
    "C": [
     16.09468490639728,
     27.289027654821602
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.10121908866995075
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JevOne",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 47.580446331109215,
    "I_sealed": 44.329708526348654,
    "base": 45.955077428728934,
    "excess": 0.6773501623223175,
    "gap": 3.2507378047605613,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 80.11686099440308,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 77.35136835482857,
       "judge": 84.17930125247197,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 20.761214630779854,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 41.86326336814473,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.82060606060606,
       "hard": 35.2175,
       "judge": 43.91203389830509,
       "standard": 46.102962962962955
      }
     },
     "sealed|choice": {
      "cc": 71.7077246661197,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 82.70637794693668,
       "hard": 65.89366515837104,
       "judge": 73.9292364990689,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 13.421236874491711,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": -8.737864077669899,
       "judge": 13.157894736842103,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 47.86016403843456,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.93652482269504,
       "hard": 43.95980392156864,
       "judge": 49.40958083832332,
       "standard": 45.29857868020299
      }
     }
    }
   },
   "jevbench_score": 31.13083884122018,
   "key": "jevone",
   "last_measured_on": "2026-10-02",
   "licence": "Component-specific JevOne/Qwen/SGLang terms recorded in RESULT.md",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9572649572649573,
    "decisive_rate": 0.624,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.76581828544445,
     "cost": 39.84212731591276,
     "intelligence": 45.955077428728934,
     "speed": 89.83885080776159
    },
    "board_release": "v1.6.43",
    "capability": 68.8604478570867,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      23.186488200616,
      37.838876804389514
     ],
     "B": [
      21.60212409324276,
      36.562829588228304
     ],
     "C": [
      16.101727917094447,
      27.301863296622916
     ]
    },
    "jevbench_score": 31.14510331640707,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 67,
    "rank": 98,
    "ranks": {
     "A": 98,
     "B": 96,
     "C": 100,
     "capability": 85
    },
    "scores": {
     "A": 31.14510331640707,
     "B": 29.58577784402432,
     "C": 21.62854396972713
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2735971907153726,
     "p50_s_raw": 0.06179859535768628,
     "p95_s_adjusted": 0.3793178545311093,
     "p95_s_raw": 0.11465892726555464
    },
    "views": {
     "Balanced 33:33:33": 27.74831039177889,
     "Emphasis on Accuracy 60:20:20": 26.4196991707309,
     "Emphasis on Cost 20:20:60": 24.78908517771933,
     "Emphasis on Speed 20:60:20": 33.418202927958156,
     "Intelligence only": 24.64935315350002
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 112,
   "ranked": true,
   "ranks": {
    "A": 112,
    "B": 110,
    "C": 115,
    "capability": 99
   },
   "repo": "https://huggingface.co/juspay/jev-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 31.13083884122018,
    "B": 29.57547920187639,
    "C": 21.618638084180684
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2804038129746914,
    "p50_s_raw": 0.0652019064873457,
    "p95_s_adjusted": 0.3924056202173233,
    "p95_s_raw": 0.12120281010866166
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.6-35B-A3B BF16 with JevOne bidirectional option-logit mapping",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.59503348121109,
      91.11403729704442
     ],
     "I": [
      40.574264452604204,
      50.931916001637504
     ],
     "capability": [
      63.974665483283545,
      69.89087834774071
     ],
     "composite": {
      "A": [
       23.176346265212082,
       37.82109096021405
      ],
      "B": [
       21.595080696244864,
       36.549542985952485
      ],
      "C": [
       16.09468490639728,
       27.289027654821602
      ]
     },
     "seed": 50
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.76581828544445,
     "capability": 68.8604478570867,
     "intelligence": 45.955077428728934
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7946666666666666,
      "calibration": 93.10943750000003,
      "cc": 73.74079933309964,
      "cc_by_type": {
       "choice": 73.74079933309964
      },
      "cc_tiered": 75.91229283026139,
      "cc_tiered_open": 80.11686099440308,
      "cc_tiered_sealed": 71.7077246661197,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.74079933309964
     },
     "noul": {
      "accuracy": 0.5973333333333334,
      "calibration": 92.18224,
      "cc": 19.466666666666676,
      "cc_by_type": {
       "noul": 19.466666666666676
      },
      "cc_tiered": 17.091225752635783,
      "cc_tiered_open": 20.761214630779854,
      "cc_tiered_sealed": 13.421236874491711,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 19.466666666666676
     },
     "score": {
      "accuracy": 0.6,
      "calibration": 90.00577735633334,
      "cc": 46.965815789473666,
      "cc_by_type": {
       "score": 46.965815789473666
      },
      "cc_tiered": 44.861713703289645,
      "cc_tiered_open": 41.86326336814473,
      "cc_tiered_sealed": 47.86016403843456,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 46.965815789473666
     }
    },
    "raw_on_A": {
     "calibration": 86.03357993512108,
     "capability": 69.68729793470257,
     "intelligence": 53.34101593428407
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.733214780833546,
    "Emphasis on Accuracy 60:20:20": 26.411486471871754,
    "Emphasis on Cost 20:20:60": 24.781854824833474,
    "Emphasis on Speed 20:60:20": 33.37881705518823,
    "Intelligence only": 24.64935315350002
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mikhail Gribov / @MihailGGribov (mihail.gribov.rs@gmail.com)",
   "axes": {
    "calibration": 77.35692434429644,
    "cost": 64.79986008365586,
    "intelligence": 35.43137209354293,
    "speed": 92.811325470547
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10031579026354083,
      "ece_score": 79.93684194729184,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 79.93684194729184,
      "score_before_time_limit": 79.93684194729184,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.22666666666666668,
      "brier": 0.19319243660768046,
      "ece": 0.16531579253832934,
      "n": 375,
      "score": 66.93684149233412,
      "score_before_time_limit": 66.93684149233412,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9893333333333333,
      "interval90_width": 0.7702222222222224,
      "n": 375,
      "nrps": 0.1175329689730567,
      "score": 85.19708959326337,
      "score_before_time_limit": 85.19708959326337,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.08926261958083807
     }
    },
    "score": 77.35692434429644
   },
   "capability": 56.39414821891969,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     19.06433443779482,
     39.60580828732759
    ],
    "B": [
     16.307275078213234,
     35.622376509927456
    ],
    "C": [
     13.239121137357513,
     27.504033532866384
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights). NO JUDGEMENT OF OURS: pricing_v15.reference() resolves Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the frozen 25 Sep snapshot (the same rate aplomb-1 carries for the same base) x this row's own measured tokens. Output tokens are 0 in every answered record: one prefill, option-label readout, nothing generated. Estimated, not charged.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.029810319999999994
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014905206499661474
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "TypeCastLM 1.4.0 (Mikhail Gribov, Qwen3.5-4B computed head)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. A FOLLOW-UP OF THE PUBLISHED typecastlm ROW: the author asks to 'measure 1.4.0 to replace it' and to list it as 'TypeCastLM'; measured here as a CANDIDATE - replace-or-keep-both is the release lane's decision. Weights mihailgribov/typecastlm-qwen3.5-3.8b tag v1.4.0 = commit 0984c3e832f7c28495134bc1d22c66bb0e0a555d (GGUF files excluded), package typecastlm[server]==1.2.2 (latest on PyPI) + flash-linear-attention + fla-core, the model card's own install lines, in an isolated venv (torch 2.14.1+cu130, transformers 5.19.0). Served with the author's own `typecastlm-serve --model <local v1.4.0 snapshot> --host 127.0.0.1 --port 8000 --device cuda`; the UNCHANGED typesafe adapter with the published row's own wire model; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE 23 LONG ITEMS ARE ANSWERED, folded in the middle to the server's own 32,768-token state context (its documented behaviour; max input 32,953 tokens observed). One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 34CA4AED",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 38.13170692188825,
    "I_sealed": 32.731037265197614,
    "base": 35.43137209354293,
    "excess": 2.8272820142523947,
    "gap": 5.4006696566906385,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 52.552512469264286,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 43.378420887071414,
       "judge": 60.44825313117996,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 25.505175983436857,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 36.33743231296363,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 64.8527119217414,
       "hard": 32.894905449875765,
       "judge": 45.884441468733286,
       "standard": 14.644332501095958
      }
     },
     "sealed|choice": {
      "cc": 47.566937838880506,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 42.986425339366505,
       "judge": 49.72067039106147,
       "standard": 44.57445498214066
      }
     },
     "sealed|noul": {
      "cc": 19.912391055582344,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": -6.796116504854366,
       "judge": 39.47368421052631,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 30.71378290112999,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.96711799317693,
       "hard": 27.922053490937206,
       "judge": 28.3107755484573,
       "standard": 28.775085204501117
      }
     }
    }
   },
   "jevbench_score": 29.8202280532983,
   "key": "typecastlm-1.4.0",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "mihailgribov/typecastlm-qwen3.5-3.8b@0984c3e832f7c28495134bc1d22c66bb0e0a555d",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8068965517241379,
    "decisive_rate": 0.7733333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.35692434429644,
     "cost": 64.79986008365586,
     "intelligence": 35.43137209354293,
     "speed": 92.95077486551261
    },
    "board_release": "v1.6.43",
    "capability": 56.39414821891969,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 78,
    "capability_rank": 104,
    "composite_ci95": {
     "A": [
      19.06853775916415,
      39.61576669827309
     ],
     "B": [
      16.309735285734533,
      35.62882083236784
     ],
     "C": [
      13.242040110530658,
      27.510949096022983
     ]
    },
    "jevbench_score": 29.827386072784904,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 70,
    "rank": 102,
    "ranks": {
     "A": 102,
     "B": 103,
     "C": 108,
     "capability": 159
    },
    "scores": {
     "A": 29.827386072784904,
     "B": 26.272910136782546,
     "C": 20.713462550545067
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21072090499219484,
     "p50_s_raw": 0.030360452496097423,
     "p95_s_adjusted": 0.2405552782700397,
     "p95_s_raw": 0.04527763913501986
    },
    "views": {
     "Balanced 33:33:33": 27.685082169472313,
     "Emphasis on Accuracy 60:20:20": 22.64780945981884,
     "Emphasis on Cost 20:20:60": 29.441987709970327,
     "Emphasis on Speed 20:60:20": 33.066455632229754,
     "Intelligence only": 17.791964524810385
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 116,
   "ranked": true,
   "ranks": {
    "A": 116,
    "B": 117,
    "C": 122,
    "capability": 176
   },
   "repo": "https://huggingface.co/mihailgribov/typecastlm-qwen3.5-3.8b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 29.8202280532983,
    "B": 26.26846689839439,
    "C": 20.70849170367937
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2110814839892555,
    "p50_s_raw": 0.03054074199462775,
    "p95_s_adjusted": 0.2479803759983042,
    "p95_s_raw": 0.048990187999152106
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.77140344619215,
      79.65897110167947
     ],
     "I": [
      29.58940031351787,
      39.89308222540656
     ],
     "capability": [
      51.45876220790242,
      59.168420588826265
     ],
     "composite": {
      "A": [
       19.06433443779482,
       39.60580828732759
      ],
      "B": [
       16.307275078213234,
       35.622376509927456
      ],
      "C": [
       13.239121137357513,
       27.504033532866384
      ]
     },
     "seed": 147
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.35692434429644,
     "capability": 56.39414821891969,
     "intelligence": 35.43137209354293
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6026666666666667,
      "calibration": 79.93684194729184,
      "cc": 49.186741566647385,
      "cc_by_type": {
       "choice": 49.186741566647385
      },
      "cc_tiered": 50.059725154072396,
      "cc_tiered_open": 52.552512469264286,
      "cc_tiered_sealed": 47.566937838880506,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.186741566647385
     },
     "noul": {
      "accuracy": 0.624,
      "calibration": 66.93684149233412,
      "cc": 24.8,
      "cc_by_type": {
       "noul": 24.8
      },
      "cc_tiered": 22.708783519509602,
      "cc_tiered_open": 25.505175983436857,
      "cc_tiered_sealed": 19.912391055582344,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.8
     },
     "score": {
      "accuracy": 0.5466666666666666,
      "calibration": 85.19708959326337,
      "cc": 33.379404446849435,
      "cc_by_type": {
       "score": 33.379404446849435
      },
      "cc_tiered": 33.52560760704681,
      "cc_tiered_open": 36.33743231296363,
      "cc_tiered_sealed": 30.71378290112999,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.379404446849435
     }
    },
    "raw_on_A": {
     "calibration": 73.20180288665794,
     "capability": 57.51186004997652,
     "intelligence": 41.821917213295094
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.67686034698086,
    "Emphasis on Accuracy 60:20:20": 22.644507697603423,
    "Emphasis on Cost 20:20:60": 29.436408038240202,
    "Emphasis on Speed 20:60:20": 33.045351087975924,
    "Intelligence only": 17.791964524810385
   }
  },
  {
   "adapter_id": "images/hopper/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "HopitAI",
   "axes": {
    "calibration": 88.80153061371698,
    "cost": 62.25627699896333,
    "intelligence": 33.59530941839744,
    "speed": 90.6039582415622
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.054931646080717024,
      "ece_score": 89.0136707838566,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 87.5894520513149,
      "score_before_time_limit": 89.0136707838566,
      "time_limit_failed_share": 0.016
     },
     "noul": {
      "abstention_rate": 0.43733333333333335,
      "brier": 0.1320524837128236,
      "ece": 0.03940530319869939,
      "n": 375,
      "score": 91.13633734041736,
      "score_before_time_limit": 92.11893936026013,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.6148888888888887,
      "n": 375,
      "nrps": 0.110200461495695,
      "score": 87.67880244941868,
      "score_before_time_limit": 89.10447403396208,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.053855028912531695
     }
    },
    "score": 88.80153061371698
   },
   "capability": 61.19842001605721,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     18.29569149792208,
     31.679044775604037
    ],
    "B": [
     15.522162379623921,
     27.99714751101641
    ],
    "C": [
     12.705341318001443,
     21.999336649725034
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.018118577586206896
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 22,
    "n_over_limit_answered_ok": 22,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Hopper",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 40.742066525947536,
    "I_sealed": 27.963579853032826,
    "base": 34.35282318949018,
    "excess": 10.205099030476466,
    "gap": 12.77848667291471,
    "penalty": 0.9779490096952353,
    "per_type_split": {
     "open|choice": {
      "cc": 68.23228363694746,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 62.25228059138095,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 18.554865424430645,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 35.439050516464505,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.1505221295817,
       "hard": 30.729313281528604,
       "judge": 33.331159411757824,
       "standard": 37.66462583683774
      }
     },
     "sealed|choice": {
      "cc": 56.856747897970024,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 48.585972850678715,
       "judge": 59.031657355679705,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": -22.330097087378643,
       "judge": 0.0,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 27.033991661128457,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.587460120023806,
       "hard": 21.43215746572834,
       "judge": 26.893973849966592,
       "standard": 20.67095253922381
      }
     }
    }
   },
   "jevbench_score": 26.506917075334485,
   "key": "hopper",
   "last_measured_on": "2026-10-02",
   "licence": "Component-specific terms recorded in RESULT.md; submitted adapter release and Qwen base retain their respective terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9428571428571428,
    "decisive_rate": 0.5660377358490566,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.07902805935959,
     "cost": 62.25627699896333,
     "intelligence": 34.73970870443837,
     "speed": 91.11434937226889
    },
    "board_release": "v1.6.43",
    "capability": 62.40936838189898,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 50,
    "capability_rank": 75,
    "composite_ci95": {
     "A": [
      19.749752505423817,
      34.23513789747224
     ],
     "B": [
      16.849888107534454,
      30.36407576551454
     ],
     "C": [
      13.715105906544313,
      23.774401317689065
     ]
    },
    "jevbench_score": 28.852652318563656,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 72,
    "rank": 104,
    "ranks": {
     "A": 104,
     "B": 105,
     "C": 114,
     "capability": 118
    },
    "scores": {
     "A": 28.852652318563656,
     "B": 25.218775497922103,
     "C": 20.036564110113652
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26295550270006063,
     "p50_s_raw": 0.0564777513500303,
     "p95_s_adjusted": 0.2942271867766976,
     "p95_s_raw": 0.07211359338834882
    },
    "views": {
     "Balanced 33:33:33": 25.942845868326373,
     "Emphasis on Accuracy 60:20:20": 21.285843260250505,
     "Emphasis on Cost 20:20:60": 27.444368192058636,
     "Emphasis on Speed 20:60:20": 31.034805184083435,
     "Intelligence only": 16.77021030692692
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 123,
   "ranked": true,
   "ranks": {
    "A": 123,
    "B": 127,
    "C": 139,
    "capability": 145
   },
   "repo": "https://huggingface.co/HopitAI/hopper",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 26.506917075334485,
    "B": 23.058774622545158,
    "C": 18.407581302315617
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2637434290722013,
    "p50_s_raw": 0.056871714536100626,
    "p95_s_adjusted": 0.3299305472988635,
    "p95_s_raw": 0.08996527364943174
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-4B plus HopitAI/hopper LoRA",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.30075058183441,
      88.36926572708367
     ],
     "I": [
      28.90778639701942,
      36.25273459922729
     ],
     "capability": [
      56.5453532049558,
      61.563676847829825
     ],
     "composite": {
      "A": [
       18.29569149792208,
       31.679044775604037
      ],
      "B": [
       15.522162379623921,
       27.99714751101641
      ],
      "C": [
       12.705341318001443,
       21.999336649725034
      ]
     },
     "seed": 41
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.80153061371698,
     "capability": 61.19842001605721,
     "intelligence": 33.59530941839744
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6826666666666666,
      "calibration": 87.5894520513149,
      "cc": 59.41759896933583,
      "cc_by_type": {
       "choice": 59.41759896933583
      },
      "cc_tiered": 62.54451576745874,
      "cc_tiered_open": 68.23228363694746,
      "cc_tiered_sealed": 56.856747897970024,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.41759896933583
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 91.13633734041736,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 9.277432712215322,
      "cc_tiered_open": 18.554865424430645,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.496,
      "calibration": 87.67880244941868,
      "cc": 30.07705243305412,
      "cc_by_type": {
       "score": 30.07705243305412
      },
      "cc_tiered": 31.23652108879648,
      "cc_tiered_open": 35.439050516464505,
      "cc_tiered_sealed": 27.033991661128457,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.07705243305412
     }
    },
    "raw_on_A": {
     "calibration": 86.24628252354488,
     "capability": 63.31122802105749,
     "intelligence": 40.376173518570106
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.016000000000000014,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 23.81704185488168,
    "Emphasis on Accuracy 60:20:20": 19.39287861067206,
    "Emphasis on Cost 20:20:60": 25.36536486248174,
    "Emphasis on Speed 20:60:20": 28.59506443576549,
    "Intelligence only": 15.16686871225422
   }
  },
  {
   "adapter_id": "A2 reviewed bundle",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 89.1627567462182,
    "cost": 63.26458220826236,
    "intelligence": 34.62714018309005,
    "speed": 90.38312320077796
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04836725385924713,
      "ece_score": 90.32654922815058,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 90.32654922815058,
      "score_before_time_limit": 90.32654922815058,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.477088948787062,
      "brier": 0.12273935128941174,
      "ece": 0.06563823281458846,
      "n": 371,
      "score": 86.87235343708231,
      "score_before_time_limit": 86.87235343708231,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967479674796748,
      "interval90_width": 0.43699186991869926,
      "n": 369,
      "nrps": 0.08429976663999325,
      "score": 90.28936757342169,
      "score_before_time_limit": 90.28936757342169,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.054956440945786535
     }
    },
    "score": 89.1627567462182
   },
   "capability": 61.89494846465412,
   "class": "unclassified",
   "composite_ci95": {
    "A": [
     21.327172085457253,
     38.38029328888697
    ],
    "B": [
     18.26402482996646,
     34.310146820317776
    ],
    "C": [
     14.810536170456427,
     26.652981450615954
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (I-2): measured input tokens; zero generated output tokens for signed logits readout. M2 floor uses the 25 Sep 2026 DeepInfra Qwen3.5-4B snapshot rates (USD 0.03/M input, USD 0.15/M output); the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen3.5-9B.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.016769261083743846
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Imajev-4B (RTX 5090)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX 5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 35.03758089292987,
    "I_sealed": 34.21669947325023,
    "base": 34.62714018309005,
    "excess": -1.752506222758607,
    "gap": 0.8208814196796368,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 66.47085555091229,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 58.47750865051904,
       "judge": 80.22412656558997,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0.6756383712905452,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": -30.000000000000004,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 37.966248756586765,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.45615279420953,
       "hard": 43.92313144450569,
       "judge": 33.905546551444175,
       "standard": 24.898584669651413
      }
     },
     "sealed|choice": {
      "cc": 62.38689563101728,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 53.676470588235276,
       "judge": 64.61824953445065,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -30.097087378640776,
       "judge": 0.0,
       "standard": 7.042253521126751
      }
     },
     "sealed|score": {
      "cc": 40.26320278873341,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.176956160290466,
       "hard": 35.01688311771597,
       "judge": 46.365402771352585,
       "standard": 30.14566547106101
      }
     }
    }
   },
   "jevbench_score": 28.648655708476518,
   "key": "imajev-4b-rtx5090-a2",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9639175257731959,
    "decisive_rate": 0.522911051212938,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.1627567462182,
     "cost": 63.26458220826236,
     "intelligence": 34.62714018309005,
     "speed": 90.51878000442085
    },
    "board_release": "v1.6.43",
    "capability": 61.89494846465412,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 51,
    "capability_rank": 76,
    "composite_ci95": {
     "A": [
      21.33216708823923,
      38.39023119963257
     ],
     "B": [
      18.266955184224454,
      34.31649986004929
     ],
     "C": [
      14.814004922388358,
      26.659882777522615
     ]
    },
    "jevbench_score": 28.655751132526984,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 73,
    "rank": 105,
    "ranks": {
     "A": 105,
     "B": 106,
     "C": 115,
     "capability": 119
    },
    "scores": {
     "A": 28.655751132526984,
     "B": 25.0248976556334,
     "C": 19.899827175365957
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29240404299926015,
     "p50_s_raw": 0.07120202149963006,
     "p95_s_adjusted": 0.3034859753679484,
     "p95_s_raw": 0.07674298768397421
    },
    "views": {
     "Balanced 33:33:33": 25.816712856140082,
     "Emphasis on Accuracy 60:20:20": 21.130055842968158,
     "Emphasis on Cost 20:20:60": 27.454793753723415,
     "Emphasis on Speed 20:60:20": 30.81252811725961,
     "Intelligence only": 16.607714361099795
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 119,
   "ranked": true,
   "ranks": {
    "A": 119,
    "B": 120,
    "C": 132,
    "capability": 135
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/110",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 28.648655708476518,
    "B": 25.020568314798886,
    "C": 19.894899797553133
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.29422962296521293,
    "p50_s_raw": 0.07211481148260646,
    "p95_s_adjusted": 0.31117255417630074,
    "p95_s_raw": 0.08058627708815037
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.2060903767957,
      89.78727704768245
     ],
     "I": [
      30.707360525579823,
      39.19772341159948
     ],
     "capability": [
      58.11557152205388,
      63.38857993223046
     ],
     "composite": {
      "A": [
       21.327172085457253,
       38.38029328888697
      ],
      "B": [
       18.26402482996646,
       34.310146820317776
      ],
      "C": [
       14.810536170456427,
       26.652981450615954
      ]
     },
     "seed": 42
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.1627567462182,
     "capability": 61.89494846465412,
     "intelligence": 34.62714018309005
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7146666666666667,
      "calibration": 90.32654922815058,
      "cc": 63.50994193041121,
      "cc_by_type": {
       "choice": 63.50994193041121
      },
      "cc_tiered": 64.4288755909648,
      "cc_tiered_open": 66.47085555091229,
      "cc_tiered_sealed": 62.38689563101728,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.50994193041121
     },
     "noul": {
      "accuracy": 0.49866666666666665,
      "calibration": 86.87235343708231,
      "cc": -0.2666666666666706,
      "cc_by_type": {
       "noul": -0.2666666666666706
      },
      "cc_tiered": 0.3378191856452726,
      "cc_tiered_open": 0.6756383712905452,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 90.28936757342169,
      "cc": 40.37192786146962,
      "cc_by_type": {
       "score": 40.37192786146962
      },
      "cc_tiered": 39.11472577266009,
      "cc_tiered_open": 37.966248756586765,
      "cc_tiered_sealed": 40.26320278873341,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 40.37192786146962
     }
    },
    "raw_on_A": {
     "calibration": 87.91523763851959,
     "capability": 61.35771088116867,
     "intelligence": 34.80018412381776
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 25.80903440282294,
    "Emphasis on Accuracy 60:20:20": 21.126969173393316,
    "Emphasis on Cost 20:20:60": 27.4495829317791,
    "Emphasis on Speed 20:60:20": 30.792846955590043,
    "Intelligence only": 16.607714361099795
   }
  },
  {
   "adapter_id": "images/malkuth-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "newfull5 (dhtocks)",
   "axes": {
    "calibration": 88.43741591513763,
    "cost": 55.82433433950372,
    "intelligence": 33.840703805987175,
    "speed": 89.3735021970277
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0565084690553745,
      "ece_score": 88.6983061889251,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.6983061889251,
      "score_before_time_limit": 88.6983061889251,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3504043126684636,
      "brier": 0.13494345407008085,
      "ece": 0.06141401617250676,
      "n": 371,
      "score": 87.71719676549866,
      "score_before_time_limit": 87.71719676549866,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.926829268292683,
      "interval90_width": 0.5194218608852753,
      "n": 369,
      "nrps": 0.1123930174593495,
      "score": 88.89674479098915,
      "score_before_time_limit": 88.89674479098915,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.054836043360433705
     }
    },
    "score": 88.43741591513763
   },
   "capability": 61.139059860562405,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     17.559605150612903,
     34.56989023188707
    ],
    "B": [
     15.040009571775245,
     31.01767193720591
    ],
    "C": [
     12.194170243481183,
     24.006868216588245
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02968398399014778
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Malkuth-4B (newfull5, Kev post-train)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 36.78555463394936,
    "I_sealed": 30.895852978024994,
    "base": 33.840703805987175,
    "excess": 3.316314013486122,
    "gap": 5.889701655924366,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 57.213904436116636,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 47.153192827933324,
       "judge": 68.35860250494396,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 19.189786059351285,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 33.95297340638018,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.45848484848485,
       "hard": 32.323055555555534,
       "judge": 29.900169491525418,
       "standard": 34.53925925925927
      }
     },
     "sealed|choice": {
      "cc": 53.38404633359997,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 43.49547511312216,
       "judge": 51.58286778398511,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 12.571668333969063,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": -16.504854368932044,
       "judge": 26.315789473684205,
       "standard": 32.3943661971831
      }
     },
     "sealed|score": {
      "cc": 26.73184426650596,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.09113475177305,
       "hard": 12.109877450980411,
       "judge": 28.95095808383231,
       "standard": 34.467461928933965
      }
     }
    }
   },
   "jevbench_score": 26.19081986039867,
   "key": "malkuth-4b",
   "last_measured_on": "2026-10-02",
   "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.921161825726141,
    "decisive_rate": 0.6495956873315364,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.43741591513763,
     "cost": 55.82433433950372,
     "intelligence": 33.840703805987175,
     "speed": 89.51571180654442
    },
    "board_release": "v1.6.43",
    "capability": 61.139059860562405,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 58,
    "capability_rank": 83,
    "composite_ci95": {
     "A": [
      17.563735957847147,
      34.57907875172623
     ],
     "B": [
      15.042433721702132,
      31.023588122626016
     ],
     "C": [
      12.197038859616075,
      24.013249133143216
     ]
    },
    "jevbench_score": 26.197476126697616,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 76,
    "rank": 109,
    "ranks": {
     "A": 109,
     "B": 112,
     "C": 122,
     "capability": 128
    },
    "scores": {
     "A": 26.197476126697616,
     "B": 23.020714347323178,
     "C": 18.19269175465112
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2985552769736387,
     "p50_s_raw": 0.07427763848681934,
     "p95_s_adjusted": 0.3744588227418717,
     "p95_s_raw": 0.11222941137093585
    },
    "views": {
     "Balanced 33:33:33": 23.437149264983223,
     "Emphasis on Accuracy 60:20:20": 19.45371080344073,
     "Emphasis on Cost 20:20:60": 24.246787312645555,
     "Emphasis on Speed 20:60:20": 28.28433154980761,
     "Intelligence only": 15.501658014110385
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 124,
   "ranked": true,
   "ranks": {
    "A": 124,
    "B": 128,
    "C": 140,
    "capability": 146
   },
   "repo": "https://github.com/newfull5/malkuth",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 26.19081986039867,
    "B": 23.016602167139798,
    "C": 18.188069347499074
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3005055939778686,
    "p50_s_raw": 0.07525279698893428,
    "p95_s_adjusted": 0.3844122505222912,
    "p95_s_raw": 0.11720612526114563
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.33637885013427,
      89.07368375555646
     ],
     "I": [
      28.800746942905594,
      38.02075761880322
     ],
     "capability": [
      56.551766449658025,
      62.63523991792939
     ],
     "composite": {
      "A": [
       17.559605150612903,
       34.56989023188707
      ],
      "B": [
       15.040009571775245,
       31.01767193720591
      ],
      "C": [
       12.194170243481183,
       24.006868216588245
      ]
     },
     "seed": 64
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.43741591513763,
     "capability": 61.139059860562405,
     "intelligence": 33.840703805987175
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6466666666666666,
      "calibration": 88.6983061889251,
      "cc": 54.813713138126026,
      "cc_by_type": {
       "choice": 54.813713138126026
      },
      "cc_tiered": 55.2989753848583,
      "cc_tiered_open": 57.213904436116636,
      "cc_tiered_sealed": 53.38404633359997,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.813713138126026
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 87.71719676549866,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 15.880727196660175,
      "cc_tiered_open": 19.189786059351285,
      "cc_tiered_sealed": 12.571668333969063,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 88.89674479098915,
      "cc": 29.251991228070175,
      "cc_by_type": {
       "score": 29.251991228070175
      },
      "cc_tiered": 30.34240883644307,
      "cc_tiered_open": 33.95297340638018,
      "cc_tiered_sealed": 26.73184426650596,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.251991228070175
     }
    },
    "raw_on_A": {
     "calibration": 85.35554104202156,
     "capability": 62.08736671102251,
     "intelligence": 38.819192380023466
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 23.430046314102366,
    "Emphasis on Accuracy 60:20:20": 19.45077415826461,
    "Emphasis on Cost 20:20:60": 24.242225485573762,
    "Emphasis on Speed 20:60:20": 28.265717566588606,
    "Intelligence only": 15.501658014110385
   }
  },
  {
   "adapter_id": "images/jqv/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "hjmurmur (Octalab)",
   "axes": {
    "calibration": 89.87138320201578,
    "cost": 51.172592607797476,
    "intelligence": 33.85815624201783,
    "speed": 82.47453178010326
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06281808353596482,
      "ece_score": 87.43638329280702,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.43638329280702,
      "score_before_time_limit": 87.43638329280702,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.34231805929919135,
      "brier": 0.11806081980479606,
      "ece": 0.04224066622806287,
      "n": 371,
      "score": 91.55186675438742,
      "score_before_time_limit": 91.55186675438742,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.959349593495935,
      "interval90_width": 0.5268744354110209,
      "n": 369,
      "nrps": 0.10409415210041804,
      "score": 90.6258995588529,
      "score_before_time_limit": 90.6258995588529,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04169392836126208
     }
    },
    "score": 89.87138320201578
   },
   "capability": 61.8647697220168,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     15.487583183427724,
     35.836921968629966
    ],
    "B": [
     13.292919140350357,
     32.64020508287098
    ],
    "C": [
     10.755266099602583,
     24.886751367104157
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.042420935960591134
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jqv (Qwen3-32B zero-shot)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 33.07530766335527,
    "I_sealed": 34.641004820680386,
    "base": 33.85815624201783,
    "excess": -4.139084799763362,
    "gap": -1.5656971573251184,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 58.31263706393481,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 47.153192827933324,
       "judge": 64.40342781806196,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 11.77087646652864,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": -19.999999999999996,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 29.142409459602362,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 31.012596729013953,
       "hard": 34.8410871612874,
       "judge": 43.10234913255198,
       "standard": -4.129949087897944
      }
     },
     "sealed|choice": {
      "cc": 53.61923821279086,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 62.75219865494053,
       "hard": 49.60407239819003,
       "judge": 55.30726256983241,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 17.979675703685576,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": -4.854368932038833,
       "judge": 21.052631578947366,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 32.32410054556472,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.65739446915423,
       "hard": 22.31408478189991,
       "judge": 37.34025956092481,
       "standard": 34.65324658805946
      }
     }
    }
   },
   "jevbench_score": 25.359236700104784,
   "key": "jqv",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (Qwen3-32B weights); serving code public",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9385245901639344,
    "decisive_rate": 0.6576819407008087,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.87138320201578,
     "cost": 51.172592607797476,
     "intelligence": 33.85815624201783,
     "speed": 83.1032423977654
    },
    "board_release": "v1.6.43",
    "capability": 61.8647697220168,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 52,
    "capability_rank": 77,
    "composite_ci95": {
     "A": [
      15.505551311407853,
      35.88495273799785
     ],
     "B": [
      13.30350456735459,
      32.67206876334683
     ],
     "C": [
      10.767743966255452,
      24.920106068054068
     ]
    },
    "jevbench_score": 25.391439220985372,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 77,
    "rank": 110,
    "ranks": {
     "A": 110,
     "B": 113,
     "C": 125,
     "capability": 120
    },
    "scores": {
     "A": 25.391439220985372,
     "B": 22.528313466257792,
     "C": 17.63294390346206
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3448032220825553,
     "p50_s_raw": 0.09740161104127765,
     "p95_s_adjusted": 1.4193988358601928,
     "p95_s_raw": 0.6346994179300964
    },
    "views": {
     "Balanced 33:33:33": 22.511071395625475,
     "Emphasis on Accuracy 60:20:20": 19.077648593308684,
     "Emphasis on Cost 20:20:60": 22.883252890033052,
     "Emphasis on Speed 20:60:20": 26.917672025192456,
     "Intelligence only": 15.525654079176814
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 125,
   "ranked": true,
   "ranks": {
    "A": 125,
    "B": 130,
    "C": 143,
    "capability": 136
   },
   "repo": "https://github.com/Octalab-Inc/jqv",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 25.359236700104784,
    "B": 22.50802625014765,
    "C": 17.61058104173943
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3853486230131239,
    "p50_s_raw": 0.11767431150656193,
    "p95_s_adjusted": 1.4678883056156335,
    "p95_s_raw": 0.6589441528078168
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3-32B, bf16, read as a direct-logit classifier (no fine-tuning)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.42862757082423,
      89.91697762368707
     ],
     "I": [
      27.681410439375405,
      39.196898115223256
     ],
     "capability": [
      56.87716097755697,
      63.63771518645792
     ],
     "composite": {
      "A": [
       15.487583183427724,
       35.836921968629966
      ],
      "B": [
       13.292919140350357,
       32.64020508287098
      ],
      "C": [
       10.755266099602583,
       24.886751367104157
      ]
     },
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.87138320201578,
     "capability": 61.8647697220168,
     "intelligence": 33.85815624201783
    },
    "per_type": {
     "choice": {
      "accuracy": 0.648,
      "calibration": 87.43638329280702,
      "cc": 54.98422742817084,
      "cc_by_type": {
       "choice": 54.98422742817084
      },
      "cc_tiered": 55.96593763836283,
      "cc_tiered_open": 58.31263706393481,
      "cc_tiered_sealed": 53.61923821279086,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.98422742817084
     },
     "noul": {
      "accuracy": 0.6106666666666667,
      "calibration": 91.55186675438742,
      "cc": 22.13333333333334,
      "cc_by_type": {
       "noul": 22.13333333333334
      },
      "cc_tiered": 14.875276085107107,
      "cc_tiered_open": 11.77087646652864,
      "cc_tiered_sealed": 17.979675703685576,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 22.13333333333334
     },
     "score": {
      "accuracy": 0.5066666666666667,
      "calibration": 90.6258995588529,
      "cc": 32.28871510053675,
      "cc_by_type": {
       "score": 32.28871510053675
      },
      "cc_tiered": 30.73325500258354,
      "cc_tiered_open": 29.142409459602362,
      "cc_tiered_sealed": 32.32410054556472,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.28871510053675
     }
    },
    "raw_on_A": {
     "calibration": 85.0933345598925,
     "capability": 62.946394387426395,
     "intelligence": 40.79945421496029
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.477331322119706,
    "Emphasis on Accuracy 60:20:20": 19.063098201361726,
    "Emphasis on Cost 20:20:60": 22.862321674327664,
    "Emphasis on Speed 20:60:20": 26.830985471359124,
    "Intelligence only": 15.525654079176814
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kirp",
   "axes": {
    "calibration": 89.9313809717458,
    "cost": 33.627578514713974,
    "intelligence": 51.42208418374838,
    "speed": 90.38067871211373
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.074450619123307,
      "ece_score": 85.1098761753386,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.1098761753386,
      "score_before_time_limit": 85.1098761753386,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.25876010781671155,
      "brier": 0.09167755563129923,
      "ece": 0.04206042293296505,
      "n": 371,
      "score": 91.58791541340699,
      "score_before_time_limit": 91.58791541340699,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.3337850045167119,
      "n": 369,
      "nrps": 0.06863762470007446,
      "score": 93.09635132649183,
      "score_before_time_limit": 93.09635132649183,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03471767438504456
     }
    },
    "score": 89.9313809717458
   },
   "capability": 70.67673257774709,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     20.22447178378376,
     25.85611310149331
    ],
    "B": [
     19.514626712083142,
     25.722339274014782
    ],
    "C": [
     14.044772072072053,
     22.374012546290665
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter qwen/qwen3.5-35b-a3b in the frozen 25 Sep snapshot (exact declared base; no explicit listing-decision line in the frozen module: release-lane line), USD 0.3125 per 1M input and USD 1.25 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.160584375
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.16308501184834123
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JPT-35B-A3B (Qwen3.5-35B-A3B fine-tune)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 52.77555274931294,
    "I_sealed": 50.06861561818382,
    "base": 51.42208418374838,
    "excess": 0.13354948869087968,
    "gap": 2.7069371311291235,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 71.66771648987253,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 71.68921044353571,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 33.899930986887505,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 13.043478260869556,
       "judge": 30.000000000000004,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 52.759010771178794,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.68236487691772,
       "hard": 47.969282366711376,
       "judge": 51.97130113741173,
       "standard": 56.55835497789477
      }
     },
     "sealed|choice": {
      "cc": 64.13898950109635,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 59.27601809954751,
       "judge": 64.61824953445065,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 31.905895053509607,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 14.563106796116498,
       "judge": 26.315789473684205,
       "standard": 54.92957746478873
      }
     },
     "sealed|score": {
      "cc": 54.16096229994551,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.36028549114205,
       "hard": 51.20129834773479,
       "judge": 55.62940093493124,
       "standard": 51.77797065629009
      }
     }
    }
   },
   "jevbench_score": 25.351627976061703,
   "key": "jpt-35b-a3b",
   "last_measured_on": "2026-10-06",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "kirp/jpt-35b-a3b@f9b45a14df4f7eaf5551f1ab26607237604a8588",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9381818181818182,
    "decisive_rate": 0.7412398921832885,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.9313809717458,
     "cost": 33.627578514713974,
     "intelligence": 51.42208418374838,
     "speed": 90.51950606084544
    },
    "board_release": "v1.6.43",
    "capability": 70.67673257774709,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.22909286309626,
      25.862384694814303
     ],
     "B": [
      19.5180684428127,
      25.72730452057616
     ],
     "C": [
      14.047981154927955,
      22.379413062160616
     ]
    },
    "jevbench_score": 25.357657195103958,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 78,
    "rank": 111,
    "ranks": {
     "A": 111,
     "B": 107,
     "C": 121,
     "capability": 73
    },
    "scores": {
     "A": 25.357657195103958,
     "B": 24.9082855667434,
     "C": 18.62541574870352
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29123902405845004,
     "p50_s_raw": 0.070619512029225,
     "p95_s_adjusted": 0.30464904906693846,
     "p95_s_raw": 0.07732452453346922
    },
    "views": {
     "Balanced 33:33:33": 22.529266440821832,
     "Emphasis on Accuracy 60:20:20": 22.81579765829606,
     "Emphasis on Cost 20:20:60": 18.893073743976853,
     "Emphasis on Speed 20:60:20": 27.47147293414375,
     "Intelligence only": 23.25952583784173
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 126,
   "ranked": true,
   "ranks": {
    "A": 126,
    "B": 121,
    "C": 138,
    "capability": 86
   },
   "repo": "https://huggingface.co/kirp/jpt-35b-a3b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 25.351627976061703,
    "B": 24.903631393069304,
    "C": 18.620987235831137
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2935272410162725,
    "p50_s_raw": 0.07176362050813623,
    "p95_s_adjusted": 0.3120927760901395,
    "p95_s_raw": 0.08104638804506976
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-35B-A3B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.28164810054457,
      90.58368457070935
     ],
     "I": [
      45.56151371729817,
      55.912127823531534
     ],
     "capability": [
      65.85556031844814,
      72.40411689862715
     ],
     "composite": {
      "A": [
       20.22447178378376,
       25.85611310149331
      ],
      "B": [
       19.514626712083142,
       25.722339274014782
      ],
      "C": [
       14.044772072072053,
       22.374012546290665
      ]
     },
     "seed": 58
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.9313809717458,
     "capability": 70.67673257774709,
     "intelligence": 51.42208418374838
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7386666666666667,
      "calibration": 85.1098761753386,
      "cc": 66.57919915121775,
      "cc_by_type": {
       "choice": 66.57919915121775
      },
      "cc_tiered": 67.90335299548444,
      "cc_tiered_open": 71.66771648987253,
      "cc_tiered_sealed": 64.13898950109635,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.57919915121775
     },
     "noul": {
      "accuracy": 0.688,
      "calibration": 91.58791541340699,
      "cc": 37.59999999999999,
      "cc_by_type": {
       "noul": 37.59999999999999
      },
      "cc_tiered": 32.902913020198554,
      "cc_tiered_open": 33.899930986887505,
      "cc_tiered_sealed": 31.905895053509607,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 37.59999999999999
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 93.09635132649183,
      "cc": 54.13323074000276,
      "cc_by_type": {
       "score": 54.13323074000276
      },
      "cc_tiered": 53.45998653556215,
      "cc_tiered_open": 52.759010771178794,
      "cc_tiered_sealed": 54.16096229994551,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.13323074000276
     }
    },
    "raw_on_A": {
     "calibration": 89.37706553915206,
     "capability": 72.5236455910987,
     "intelligence": 55.67022564304535
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.5229210761636,
    "Emphasis on Accuracy 60:20:20": 22.811892550347967,
    "Emphasis on Cost 20:20:60": 18.890395934033226,
    "Emphasis on Speed 20:60:20": 27.454496254659674,
    "Intelligence only": 23.25952583784173
   }
  },
  {
   "adapter_id": "images/decider-35b-a3b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 83.61830413355554,
    "cost": 34.38579876859529,
    "intelligence": 48.72471958402083,
    "speed": 91.16871958052303
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10659875000000013,
      "ece_score": 78.68024999999997,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 78.68024999999997,
      "score_before_time_limit": 78.68024999999997,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2613333333333333,
      "brier": 0.11783857269333342,
      "ece": 0.07045920000000003,
      "n": 375,
      "score": 85.90816,
      "score_before_time_limit": 85.90816,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9253333333333333,
      "interval90_width": 0.40466666666666656,
      "n": 375,
      "nrps": 0.08455155198666667,
      "score": 86.26650240066667,
      "score_before_time_limit": 86.26650240066667,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09505919999999997
     }
    },
    "score": 83.61830413355554
   },
   "capability": 66.1715118587882,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     18.695770206703155,
     26.589340433662066
    ],
    "B": [
     17.88508636306868,
     26.2482990084598
    ],
    "C": [
     12.98317375465497,
     20.36566726425127
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.1538650708128079
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-35b-a3b (Mapika)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 52.787614448504826,
    "I_sealed": 44.66182471953684,
    "base": 48.72471958402083,
    "excess": 5.552402086529744,
    "gap": 8.125789728967987,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 63.96752213183363,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 60.36489462094999,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 43.862663906142174,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 50.53265730753868,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.64121212121212,
       "hard": 47.127870370370395,
       "judge": 62.23203389830508,
       "standard": 44.23888888888891
      }
     },
     "sealed|choice": {
      "cc": 58.35475838263821,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 73.3944276106718,
       "hard": 49.60407239819003,
       "judge": 60.89385474860336,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 27.440215333904295,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": 2.9126213592232997,
       "judge": 36.8421052631579,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 48.19050044206803,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.689858156028365,
       "hard": 43.85012254901962,
       "judge": 53.23419161676646,
       "standard": 42.05604060913702
      }
     }
    }
   },
   "jevbench_score": 24.76858376061229,
   "key": "decider-35b-a3b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9061371841155235,
    "decisive_rate": 0.7386666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.61830413355554,
     "cost": 34.38579876859529,
     "intelligence": 48.72471958402083,
     "speed": 91.27357492980725
    },
    "board_release": "v1.6.43",
    "capability": 66.1715118587882,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      18.69889165560131,
      26.594050353427875
     ],
     "B": [
      17.887370941224734,
      26.251970770447716
     ],
     "C": [
      12.985341427500913,
      20.369267436681472
     ]
    },
    "jevbench_score": 24.772887446577812,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 79,
    "rank": 112,
    "ranks": {
     "A": 112,
     "B": 109,
     "C": 127,
     "capability": 96
    },
    "scores": {
     "A": 24.772887446577812,
     "B": 24.135656537463312,
     "C": 17.20339406012348
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2493798840790987,
     "p50_s_raw": 0.04968994203954935,
     "p95_s_adjusted": 0.29907567272894087,
     "p95_s_raw": 0.07453783636447042
    },
    "views": {
     "Balanced 33:33:33": 22.248611212569305,
     "Emphasis on Accuracy 60:20:20": 22.101309301010737,
     "Emphasis on Cost 20:20:60": 18.91495293933302,
     "Emphasis on Speed 20:60:20": 27.229064334478497,
     "Intelligence only": 21.8839776409655
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 127,
   "ranked": true,
   "ranks": {
    "A": 127,
    "B": 123,
    "C": 145,
    "capability": 112
   },
   "repo": "https://huggingface.co/Mapika/decider-35b-a3b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 24.76858376061229,
    "B": 24.132388310727467,
    "C": 17.20040538931409
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.253978336090222,
    "p50_s_raw": 0.051989168045111,
    "p95_s_adjusted": 0.3008370825322345,
    "p95_s_raw": 0.07541854126611724
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-35B-A3B-Base with a trained decision readout, 34.7B parameters / 3B active",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.22742585943612,
      84.9038621148463
     ],
     "I": [
      43.19083908199794,
      52.63193908662482
     ],
     "capability": [
      61.40117742704215,
      67.98807044411026
     ],
     "composite": {
      "A": [
       18.695770206703155,
       26.589340433662066
      ],
      "B": [
       17.88508636306868,
       26.2482990084598
      ],
      "C": [
       12.98317375465497,
       20.36566726425127
      ]
     },
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.61830413355554,
     "capability": 66.1715118587882,
     "intelligence": 48.72471958402083
    },
    "per_type": {
     "choice": {
      "accuracy": 0.692,
      "calibration": 78.68024999999997,
      "cc": 60.611198999649474,
      "cc_by_type": {
       "choice": 60.611198999649474
      },
      "cc_tiered": 61.16114025723592,
      "cc_tiered_open": 63.96752213183363,
      "cc_tiered_sealed": 58.35475838263821,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.611198999649474
     },
     "noul": {
      "accuracy": 0.6693333333333333,
      "calibration": 85.90816,
      "cc": 33.86666666666667,
      "cc_by_type": {
       "noul": 33.86666666666667
      },
      "cc_tiered": 35.65143962002323,
      "cc_tiered_open": 43.862663906142174,
      "cc_tiered_sealed": 27.440215333904295,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.86666666666667
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 86.26650240066667,
      "cc": 48.45208771929824,
      "cc_by_type": {
       "score": 48.45208771929824
      },
      "cc_tiered": 49.36157887480336,
      "cc_tiered_open": 50.53265730753868,
      "cc_tiered_sealed": 48.19050044206803,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.45208771929824
     }
    },
    "raw_on_A": {
     "calibration": 80.25053271222222,
     "capability": 65.45225190461375,
     "intelligence": 50.65397109700527
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.24398296089557,
    "Emphasis on Accuracy 60:20:20": 22.09856876882974,
    "Emphasis on Cost 20:20:60": 18.91294561663506,
    "Emphasis on Speed 20:60:20": 27.2165893380555,
    "Intelligence only": 21.8839776409655
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "iamupd / corners-ai (iamupd@gmail.com)",
   "axes": {
    "calibration": 81.18327994577871,
    "cost": 65.17140064644872,
    "intelligence": 32.65682750200105,
    "speed": 91.2364277414064
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12863454680642747,
      "ece_score": 74.2730906387145,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 74.2730906387145,
      "score_before_time_limit": 74.2730906387145,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.444743935309973,
      "brier": 0.11602338129808766,
      "ece": 0.08236478989729613,
      "n": 371,
      "score": 83.52704202054078,
      "score_before_time_limit": 83.52704202054078,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.997289972899729,
      "interval90_width": 0.6962511291779581,
      "n": 369,
      "nrps": 0.09343875797512342,
      "score": 85.74970717808084,
      "score_before_time_limit": 85.74970717808084,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09578354923162988
     }
    },
    "score": 81.18327994577871
   },
   "capability": 56.92005372388988,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     19.62860851096809,
     33.5004384630263
    ],
    "B": [
     16.76366776722062,
     29.649200805597484
    ],
    "C": [
     13.630978132616734,
     23.264193377101595
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights). NO JUDGEMENT OF OURS: pricing_v15.reference() resolves Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the frozen 25 Sep snapshot (the same rate aplomb-1 carries for the same base) x this row's own measured tokens. Output tokens are 0 in every answered record: one prefill, option-label readout, nothing generated. Estimated, not charged.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.014264039999999997
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014486161137440754
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "CoCo-Decision-4B (corners-ai, LoRA on Qwen3.5-4B, option-label logits of one pass, served by oh-my-jev)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,477 of 1,500 answered. LoRA adapter corners-ai/CoCo-Decision-4B-Ko tag v1.0.0 = commit b580ede993493690212a07ed0e6cb972868cc808 (the submission's 'Version 1.0.0', the card's own pin; Apache-2.0) on Qwen/Qwen3.5-4B @ 851bf6e8 as its own adapter_config.json names it. Toolkit github.com/iamupd/oh-my-jev @ f0b8c786 (Apache-2.0) installed with its README's `uv sync` plus the semif extra the README names for --adapter (torch 2.9.1+cu128, transformers 5.17.0, peft 0.21.0). Served by the author's own `omj serve --backend semif --adapter <adapter dir>`: the README documents that --adapter runs locally on semif and loads the adapter's base, in 4-bit only when it would not fit; on the 32 GB card omj itself chose bf16 ('Using Qwen/Qwen3.5-4B (the adapter's base model, bf16)'), the card's headline configuration. NO config.toml was authored and no setting of ours was chosen, which removes run 58's blocker. UNCHANGED typesafe adapter; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE 23 LONG ITEMS ARE REFUSED with the documented 422 context_length_exceeded at omj serve's default 4,096-token limit (README 'Known limitations'); they are the system's own disclosed refusal, counted wrong, and no other item failed. One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 619F3346 (bh-submit 0195e1e7)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 32.635837624396466,
    "I_sealed": 32.67781737960564,
    "base": 32.65682750200105,
    "excess": -2.6153673976474203,
    "gap": -0.04197975520917652,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 56.102191211482165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 54.70273670965714,
       "judge": 68.35860250494396,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": -30.000000000000004,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 41.80532166170725,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.92275346241979,
       "hard": 29.940973271546778,
       "judge": 47.18651389666748,
       "standard": 53.90351418923157
      }
     },
     "sealed|choice": {
      "cc": 60.224217892665166,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 76.05498484960461,
       "hard": 58.76696832579185,
       "judge": 56.238361266294234,
       "standard": 61.20211848749847
      }
     },
     "sealed|noul": {
      "cc": 1.5482925144119708,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": -20.38834951456311,
       "judge": 7.8947368421052655,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 36.26094173173978,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 57.8083031267016,
       "hard": 30.22640846893988,
       "judge": 36.4425387268291,
       "standard": 37.283932067224676
      }
     }
    }
   },
   "jevbench_score": 24.642572772350576,
   "key": "coco-decision-4b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "corners-ai/CoCo-Decision-4B-Ko@b580ede993493690212a07ed0e6cb972868cc808",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9611650485436893,
    "decisive_rate": 0.555256064690027,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.18327994577871,
     "cost": 65.17140064644872,
     "intelligence": 32.65682750200105,
     "speed": 91.3615355688411
    },
    "board_release": "v1.6.43",
    "capability": 56.92005372388988,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 73,
    "capability_rank": 99,
    "composite_ci95": {
     "A": [
      19.632668233405447,
      33.50810171156304
     ],
     "B": [
      16.766044087628412,
      29.654016739790904
     ],
     "C": [
      13.633797384309343,
      23.269515077474335
     ]
    },
    "jevbench_score": 24.64791535308594,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 80,
    "rank": 113,
    "ranks": {
     "A": 113,
     "B": 122,
     "C": 128,
     "capability": 153
    },
    "scores": {
     "A": 24.64791535308594,
     "B": 21.361319353519175,
     "C": 17.116607884087454
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23897131998674012,
     "p50_s_raw": 0.044485659993370064,
     "p95_s_adjusted": 0.30584448990703095,
     "p95_s_raw": 0.07792224495351546
    },
    "views": {
     "Balanced 33:33:33": 22.48701917025063,
     "Emphasis on Accuracy 60:20:20": 18.052162943961683,
     "Emphasis on Cost 20:20:60": 24.348744693137398,
     "Emphasis on Speed 20:60:20": 27.066970501623096,
     "Intelligence only": 13.930989601396824
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 128,
   "ranked": true,
   "ranks": {
    "A": 128,
    "B": 140,
    "C": 146,
    "capability": 170
   },
   "repo": "https://huggingface.co/corners-ai/CoCo-Decision-4B-Ko",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 24.642572772350576,
    "B": 21.358108905011534,
    "C": 17.112897758576786
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.240418037987547,
    "p50_s_raw": 0.0452090189937735,
    "p95_s_adjusted": 0.3128889149724273,
    "p95_s_raw": 0.08144445748621365
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.60430858133772,
      83.74029534027535
     ],
     "I": [
      29.768152266734436,
      37.070992355300035
     ],
     "capability": [
      54.51166132245717,
      59.43802140351146
     ],
     "composite": {
      "A": [
       19.62860851096809,
       33.5004384630263
      ],
      "B": [
       16.76366776722062,
       29.649200805597484
      ],
      "C": [
       13.630978132616734,
       23.264193377101595
      ]
     },
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.18327994577871,
     "capability": 56.92005372388988,
     "intelligence": 32.65682750200105
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6946666666666667,
      "calibration": 74.2730906387145,
      "cc": 60.9522275797391,
      "cc_by_type": {
       "choice": 60.9522275797391
      },
      "cc_tiered": 58.163204552073665,
      "cc_tiered_open": 56.102191211482165,
      "cc_tiered_sealed": 60.224217892665166,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.9522275797391
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 83.52704202054078,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 0.7741462572059854,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 1.5482925144119708,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.5786666666666667,
      "calibration": 85.74970717808084,
      "cc": 37.72075365036465,
      "cc_by_type": {
       "score": 37.72075365036465
      },
      "cc_tiered": 39.033131696723515,
      "cc_tiered_open": 41.80532166170725,
      "cc_tiered_sealed": 36.26094173173978,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 37.72075365036465
     }
    },
    "raw_on_A": {
     "calibration": 81.23470206450914,
     "capability": 59.56576106652099,
     "intelligence": 37.89682006853285
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.481090287678384,
    "Emphasis on Accuracy 60:20:20": 18.04987008065395,
    "Emphasis on Cost 20:20:60": 24.344573564470792,
    "Emphasis on Speed 20:60:20": 27.051513426744904,
    "Intelligence only": 13.930989601396824
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "oraculumai",
   "axes": {
    "calibration": 84.0077997110931,
    "cost": 64.32996474661243,
    "intelligence": 32.54303915979197,
    "speed": 90.57533514329342
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07560407620955467,
      "ece_score": 84.87918475808907,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 84.87918475808907,
      "score_before_time_limit": 84.87918475808907,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.37735849056603776,
      "brier": 0.1368158290573463,
      "ece": 0.05776783172439211,
      "n": 371,
      "score": 88.44643365512158,
      "score_before_time_limit": 88.44643365512158,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9051490514905149,
      "interval90_width": 0.4345076784101176,
      "n": 369,
      "nrps": 0.12172060479348325,
      "score": 78.69778072006868,
      "score_before_time_limit": 78.69778072006868,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1521618904025716
     }
    },
    "score": 84.0077997110931
   },
   "capability": 58.27541943544253,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     17.54165637724151,
     31.010327647371174
    ],
    "B": [
     14.869191995046108,
     27.352564747360137
    ],
    "C": [
     12.181705817528828,
     21.534949755118866
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: frozen 25 Sep DeepInfra Qwen/Qwen3.5-4B market reference, already marked deprecated at that cutoff (replaced by Qwen/Qwen3.5-9B). $0.03/M input, zero generated output; 836,500 measured input tokens across 1,624 decisions. Frozen v1.5 base-model floor applied; no bookable Manchego tariff claimed. Re-score if the basis changes.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015452586206896551
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Manchego v2.1",
   "endpoint_condition": "evaluator-owned offline RTX6000 GPU",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 40.1322357986733,
    "I_sealed": 26.800244420595554,
    "base": 33.46624010963443,
    "excess": 10.758603735639502,
    "gap": 13.331991378077745,
    "penalty": 0.972413962643605,
    "per_type_split": {
     "open|choice": {
      "cc": 60.640330926520605,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 45.26580685750237,
       "judge": 68.35860250494396,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 19.02691511387163,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 13.043478260869556,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 40.72946135562766,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.415593474232956,
       "hard": 33.07729129866341,
       "judge": 27.484035794573337,
       "standard": 64.558873751835
      }
     },
     "sealed|choice": {
      "cc": 56.6033475264085,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 49.60407239819003,
       "judge": 56.238361266294234,
       "standard": 66.7446729892844
      }
     },
     "sealed|noul": {
      "cc": 2.9507869907230795,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -30.097087378640776,
       "judge": 13.157894736842103,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 20.84659874465509,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.898828557922364,
       "hard": 20.484461869906934,
       "judge": 15.149834013146535,
       "standard": 14.089904684780608
      }
     }
    }
   },
   "jevbench_score": 24.48047111848564,
   "key": "manchego-v21-v15",
   "last_measured_on": "2026-10-04",
   "licence": "Apache-2.0 (code and weights, author-declared)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9047619047619048,
    "decisive_rate": 0.6226415094339622,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.0077997110931,
     "cost": 64.32996474661243,
     "intelligence": 32.54303915979197,
     "speed": 90.7347510499745
    },
    "board_release": "v1.6.43",
    "capability": 58.27541943544253,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 64,
    "capability_rank": 89,
    "composite_ci95": {
     "A": [
      17.54626281581445,
      31.019351182791876
     ],
     "B": [
      14.871831849106172,
      27.358180563302277
     ],
     "C": [
      12.18490473320448,
      21.54121609916103
     ]
    },
    "jevbench_score": 24.487333472500133,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 81,
    "rank": 114,
    "ranks": {
     "A": 114,
     "B": 124,
     "C": 129,
     "capability": 142
    },
    "scores": {
     "A": 24.487333472500133,
     "B": 21.196520189880484,
     "C": 17.005092689236207
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2574284589994932,
     "p50_s_raw": 0.05371422949974658,
     "p95_s_adjusted": 0.32799584188469455,
     "p95_s_raw": 0.08899792094234726
    },
    "views": {
     "Balanced 33:33:33": 22.181162996647437,
     "Emphasis on Accuracy 60:20:20": 17.83638056849814,
     "Emphasis on Cost 20:20:60": 23.964666214390906,
     "Emphasis on Speed 20:60:20": 26.697557732929795,
     "Intelligence only": 13.785874409327832
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 129,
   "ranked": true,
   "ranks": {
    "A": 129,
    "B": 141,
    "C": 147,
    "capability": 160
   },
   "repo": "https://github.com/nschlaepfer/manchego-serve",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 24.48047111848564,
    "B": 21.192406356663167,
    "C": 17.00032716561503
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26150697499397213,
    "p50_s_raw": 0.055753487496986054,
    "p95_s_adjusted": 0.33495248848921616,
    "p95_s_raw": 0.09247624424460808
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server manchego-serve v0.1.1 @ 0ee62188af474e3c1d11d47a9c4c8b3a08766db3; oraculumai/Manchego @ 77403228b7dfdf823af99a5f562bdcf80b708d4c. Evaluator-owned RTX6000, offline/read-only container, safetensors with verified weight and support-file checksums, bfloat16, sequential execution, temperature 1.0 and original option order. Stock TypeSafe adapter; all three types native. The author disclosed using public aggregate/family results during development, a prior Jev-assisted phrase filter on two template pools, and one Jev-labelled development group for checkpoint selection. These disclosures do not establish item-text leakage; this is not a claim of fully independent development. Cost is an estimate using measured token usage and the frozen Qwen3.5-4B reference, not a bookable Manchego tariff.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.8981045039492,
      85.26223280687248
     ],
     "I": [
      28.50329098429543,
      35.95139455461947
     ],
     "capability": [
      54.23896847458738,
      59.77267571842297
     ],
     "composite": {
      "A": [
       17.54165637724151,
       31.010327647371174
      ],
      "B": [
       14.869191995046108,
       27.352564747360137
      ],
      "C": [
       12.181705817528828,
       21.534949755118866
      ]
     },
     "seed": 65
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.0077997110931,
     "capability": 58.27541943544253,
     "intelligence": 32.54303915979197
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6693333333333333,
      "calibration": 84.87918475808907,
      "cc": 57.712456068887754,
      "cc_by_type": {
       "choice": 57.712456068887754
      },
      "cc_tiered": 58.62183922646455,
      "cc_tiered_open": 60.640330926520605,
      "cc_tiered_sealed": 56.6033475264085,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.712456068887754
     },
     "noul": {
      "accuracy": 0.5573333333333333,
      "calibration": 88.44643365512158,
      "cc": 11.466666666666669,
      "cc_by_type": {
       "noul": 11.466666666666669
      },
      "cc_tiered": 10.988851052297356,
      "cc_tiered_open": 19.02691511387163,
      "cc_tiered_sealed": 2.9507869907230795,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 11.466666666666669
     },
     "score": {
      "accuracy": 0.49066666666666664,
      "calibration": 78.69778072006868,
      "cc": 26.44898115857113,
      "cc_by_type": {
       "score": 26.44898115857113
      },
      "cc_tiered": 30.788030050141376,
      "cc_tiered_open": 40.72946135562766,
      "cc_tiered_sealed": 20.84659874465509,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.44898115857113
     }
    },
    "raw_on_A": {
     "calibration": 84.55956989627305,
     "capability": 59.747806252757606,
     "intelligence": 34.936042609242165
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.17365589657675,
    "Emphasis on Accuracy 60:20:20": 17.833467542474036,
    "Emphasis on Cost 20:20:60": 23.959407866642515,
    "Emphasis on Speed 20:60:20": 26.677989674889247,
    "Intelligence only": 13.785874409327832
   }
  },
  {
   "adapter_id": "images/decision-4b-v12/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 88.42488519968094,
    "cost": 63.07359395281681,
    "intelligence": 32.40310972936693,
    "speed": 93.8753186824915
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0672265434886423,
      "ece_score": 86.55469130227154,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 86.55469130227154,
      "score_before_time_limit": 86.55469130227154,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.41239892183288407,
      "brier": 0.12038668872149434,
      "ece": 0.03739565815349834,
      "n": 371,
      "score": 92.52086836930033,
      "score_before_time_limit": 92.52086836930033,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.975609756097561,
      "interval90_width": 0.5718157181571816,
      "n": 369,
      "nrps": 0.10515132123479526,
      "score": 86.19909592747098,
      "score_before_time_limit": 86.19909592747098,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.08543338010789257
     }
    },
    "score": 88.42488519968094
   },
   "capability": 60.41399746452394,
   "class": "unclassified",
   "composite_ci95": {
    "A": [
     15.391366278804872,
     34.34631539785436
    ],
    "B": [
     12.887969864384408,
     30.38764592765839
    ],
    "C": [
     10.688448804725608,
     23.851607915176643
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01701689039408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 4B v1.2 (FlyMyJev, Qwen3.5-4B + LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 32.99062805982875,
    "I_sealed": 31.81559139890512,
    "base": 32.40310972936693,
    "excess": -1.398350981514613,
    "gap": 1.175036660923631,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.39828743553336,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 50.92796476879523,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 6.031055900621121,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": 30.434782608695656,
       "judge": -30.000000000000004,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 31.542540843331764,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.77190646723929,
       "hard": 38.72644314542413,
       "judge": 25.494038632486827,
       "standard": 17.13280674346067
      }
     },
     "sealed|choice": {
      "cc": 58.83327326304771,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 56.730769230769226,
       "judge": 56.238361266294234,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 8.040955184350867,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -8.737864077669899,
       "judge": 13.157894736842103,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 28.5725457493168,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.99125190199095,
       "hard": 23.870346180703184,
       "judge": 26.811123175914563,
       "standard": 25.409725670310312
      }
     }
    }
   },
   "jevbench_score": 24.461373445342794,
   "key": "decision-4b-v12",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.944954128440367,
    "decisive_rate": 0.5876010781671159,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.42488519968094,
     "cost": 63.07359395281681,
     "intelligence": 32.40310972936693,
     "speed": 93.92936299294371
    },
    "board_release": "v1.6.43",
    "capability": 60.41399746452394,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 59,
    "capability_rank": 84,
    "composite_ci95": {
     "A": [
      15.392618056749317,
      34.34955627190713
     ],
     "B": [
      12.888673039303178,
      30.38967783428378
     ],
     "C": [
      10.689318094964802,
      23.853858522157736
     ]
    },
    "jevbench_score": 24.463556699266523,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 82,
    "rank": 115,
    "ranks": {
     "A": 115,
     "B": 125,
     "C": 130,
     "capability": 130
    },
    "scores": {
     "A": 24.463556699266523,
     "B": 21.09790572169507,
     "C": 16.988581041157314
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18829862689599394,
     "p50_s_raw": 0.019149313447996974,
     "p95_s_adjusted": 0.2148901690961793,
     "p95_s_raw": 0.032445084548089656
    },
    "views": {
     "Balanced 33:33:33": 21.964945240595316,
     "Emphasis on Accuracy 60:20:20": 17.633890573157025,
     "Emphasis on Cost 20:20:60": 23.575828038111915,
     "Emphasis on Speed 20:60:20": 26.69802362301605,
     "Intelligence only": 13.608807339398107
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 130,
   "ranked": true,
   "ranks": {
    "A": 130,
    "B": 142,
    "C": 148,
    "capability": 148
   },
   "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v12",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 24.461373445342794,
    "B": 21.09660661221488,
    "C": 16.987064892599168
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18921992308460175,
    "p50_s_raw": 0.01960996154230088,
    "p95_s_adjusted": 0.2165216211695224,
    "p95_s_raw": 0.0332608105847612
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.81349929936302,
      88.39912781374252
     ],
     "I": [
      26.905961082898493,
      37.32153047837871
     ],
     "capability": [
      55.53187740606071,
      61.760259132085466
     ],
     "composite": {
      "A": [
       15.391366278804872,
       34.34631539785436
      ],
      "B": [
       12.887969864384408,
       30.38764592765839
      ],
      "C": [
       10.688448804725608,
       23.851607915176643
      ]
     },
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.42488519968094,
     "capability": 60.41399746452394,
     "intelligence": 32.40310972936693
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 86.55469130227154,
      "cc": 60.099656129515054,
      "cc_by_type": {
       "choice": 60.099656129515054
      },
      "cc_tiered": 60.11578034929053,
      "cc_tiered_open": 61.39828743553336,
      "cc_tiered_sealed": 58.83327326304771,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.099656129515054
     },
     "noul": {
      "accuracy": 0.5493333333333333,
      "calibration": 92.52086836930033,
      "cc": 9.866666666666667,
      "cc_by_type": {
       "noul": 9.866666666666667
      },
      "cc_tiered": 7.036005542485994,
      "cc_tiered_open": 6.031055900621121,
      "cc_tiered_sealed": 8.040955184350867,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 9.866666666666667
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 86.19909592747098,
      "cc": 30.992722913904714,
      "cc_by_type": {
       "score": 30.992722913904714
      },
      "cc_tiered": 30.057543296324283,
      "cc_tiered_open": 31.542540843331764,
      "cc_tiered_sealed": 28.5725457493168,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.992722913904714
     }
    },
    "raw_on_A": {
     "calibration": 81.8292718059759,
     "capability": 58.67224928043624,
     "intelligence": 35.515226754896574
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 21.962598546947525,
    "Emphasis on Accuracy 60:20:20": 17.632983029071085,
    "Emphasis on Cost 20:20:60": 23.574205862653095,
    "Emphasis on Speed 20:60:20": 26.691783807260723,
    "Intelligence only": 13.608807339398107
   }
  },
  {
   "adapter_id": "images/raw-qwen3-4b-instruct-2507/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 35.18175191334879,
    "cost": 63.36068112101963,
    "intelligence": 33.95117742100547,
    "speed": 90.72731866330435
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.44119675391398117,
      "ece_score": 11.760649217203767,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 11.760649217203767,
      "score_before_time_limit": 11.760649217203767,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.07547169811320753,
      "brier": 0.26616192586960774,
      "ece": 0.254956633051564,
      "n": 371,
      "score": 49.008673389687196,
      "score_before_time_limit": 49.008673389687196,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.5501355013550135,
      "interval90_width": 0.07452574525745256,
      "n": 369,
      "nrps": 0.1781187945424487,
      "score": 44.77593313315542,
      "score_before_time_limit": 44.77593313315542,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.46318127139722143
     }
    },
    "score": 35.18175191334879
   },
   "capability": 34.56646466717713,
   "class": "raw-logit-control",
   "composite_ci95": {
    "A": [
     13.82916739077245,
     28.930726334443104
    ],
    "B": [
     12.601646040268905,
     27.178830974590664
    ],
    "C": [
     9.6035884658142,
     20.090782176696603
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01664602832512315
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 4B Instruct 2507 direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 39.33458111901565,
    "I_sealed": 28.646049395387923,
    "base": 33.990315257201786,
    "excess": 8.115144081189486,
    "gap": 10.68853172362773,
    "penalty": 0.9988485591881051,
    "per_type_split": {
     "open|choice": {
      "cc": 53.102399069057086,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 39.60364894620951,
       "judge": 44.62755438365195,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 40.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 24.532814267285957,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.98343409680498,
       "hard": 0.7007591330399521,
       "judge": 15.311071908696839,
       "standard": 75.80422815890212
      }
     },
     "sealed|choice": {
      "cc": 41.64735120428594,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 53.44024831867564,
       "hard": 30.2601809954751,
       "judge": 41.34078212290503,
       "standard": 58.9850966867841
      }
     },
     "sealed|noul": {
      "cc": 25.067667527906327,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 0.9708737864077666,
       "judge": 31.578947368421062,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 19.223129453971513,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 48.978480535664694,
       "hard": 7.881002538538395,
       "judge": 13.283448182092084,
       "standard": 35.9392296518103
      }
     }
    }
   },
   "jevbench_score": 21.77888128460097,
   "key": "raw-qwen3-4b-instruct-2507",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7259475218658892,
    "decisive_rate": 0.9245283018867925,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 35.18175191334879,
     "cost": 63.36068112101963,
     "intelligence": 33.95117742100547,
     "speed": 91.16296741568695
    },
    "board_release": "v1.6.43",
    "capability": 34.56646466717713,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 123,
    "capability_rank": 152,
    "composite_ci95": {
     "A": [
      13.8369005003162,
      28.950164844761904
     ],
     "B": [
      12.606782226433888,
      27.192389151058098
     ],
     "C": [
      9.608958680775137,
      20.10428114219577
     ]
    },
    "jevbench_score": 21.792436038282993,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 83,
    "rank": 116,
    "ranks": {
     "A": 116,
     "B": 128,
     "C": 134,
     "capability": 219
    },
    "scores": {
     "A": 21.792436038282993,
     "B": 20.20761454160472,
     "C": 15.133636137696522
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24207517188042402,
     "p50_s_raw": 0.04603758594021201,
     "p95_s_adjusted": 0.31604797318577765,
     "p95_s_raw": 0.08302398659288883
    },
    "views": {
     "Balanced 33:33:33": 24.609772187762267,
     "Emphasis on Accuracy 60:20:20": 20.02676319678133,
     "Emphasis on Cost 20:20:60": 26.265558574780524,
     "Emphasis on Speed 20:60:20": 29.50118725011904,
     "Intelligence only": 15.653970524552685
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 133,
   "ranked": true,
   "ranks": {
    "A": 133,
    "B": 146,
    "C": 153,
    "capability": 241
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.77888128460097,
    "B": 20.19828908936253,
    "C": 15.124223114306227
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24956028824672102,
    "p50_s_raw": 0.049780144123360515,
    "p95_s_adjusted": 0.33891645497642453,
    "p95_s_raw": 0.09445822748821228
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-4B-Instruct-2507 BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      30.079848447424265,
      39.87330204447962
     ],
     "I": [
      28.52380607287802,
      37.88233031513499
     ],
     "capability": [
      29.696940965810747,
      38.2358105496228
     ],
     "composite": {
      "A": [
       13.82916739077245,
       28.930726334443104
      ],
      "B": [
       12.601646040268905,
       27.178830974590664
      ],
      "C": [
       9.6035884658142,
       20.090782176696603
      ]
     },
     "seed": 96
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 35.18175191334879,
     "capability": 34.56646466717713,
     "intelligence": 33.95117742100547
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5626666666666666,
      "calibration": 11.760649217203767,
      "cc": 44.07131286530316,
      "cc_by_type": {
       "choice": 44.07131286530316
      },
      "cc_tiered": 47.37487513667151,
      "cc_tiered_open": 53.102399069057086,
      "cc_tiered_sealed": 41.64735120428594,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 44.07131286530316
     },
     "noul": {
      "accuracy": 0.664,
      "calibration": 49.008673389687196,
      "cc": 32.800000000000004,
      "cc_by_type": {
       "noul": 32.800000000000004
      },
      "cc_tiered": 32.718098774305126,
      "cc_tiered_open": 40.36853002070393,
      "cc_tiered_sealed": 25.067667527906327,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.800000000000004
     },
     "score": {
      "accuracy": 0.456,
      "calibration": 44.77593313315542,
      "cc": 20.99121192415334,
      "cc_by_type": {
       "score": 20.99121192415334
      },
      "cc_tiered": 21.877971860628733,
      "cc_tiered_open": 24.532814267285957,
      "cc_tiered_sealed": 19.223129453971513,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 20.99121192415334
     }
    },
    "raw_on_A": {
     "calibration": 37.75992930947443,
     "capability": 38.17464116904609,
     "intelligence": 38.58935302861775
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 24.586731395456564,
    "Emphasis on Accuracy 60:20:20": 20.017603879088405,
    "Emphasis on Cost 20:20:60": 26.249805947125328,
    "Emphasis on Speed 20:60:20": 29.441653503402815,
    "Intelligence only": 15.653970524552685
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AutoTrust AI Lab",
   "axes": {
    "calibration": 90.89967561034447,
    "cost": 31.02571003473804,
    "intelligence": 56.828664559360675,
    "speed": 86.4682823414372
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.044592591608484194,
      "ece_score": 91.08148167830316,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 91.08148167830316,
      "score_before_time_limit": 91.08148167830316,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.21293800539083563,
      "brier": 0.0834318516755718,
      "ece": 0.0453231523993617,
      "n": 371,
      "score": 90.93536952012767,
      "score_before_time_limit": 90.93536952012767,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.2831978319783197,
      "n": 369,
      "nrps": 0.07252867726920081,
      "score": 90.68217563260262,
      "score_before_time_limit": 90.68217563260262,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05691390503937325
     }
    },
    "score": 90.89967561034447
   },
   "capability": 73.86417008485257,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     20.64501531235954,
     21.6223551289021
    ],
    "B": [
     20.4912109596719,
     22.005422726140193
    ],
    "C": [
     15.246303093563608,
     21.6223551289021
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.19607951999999995
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.19913289099526063
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JEV-27B (AutoTrust, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.748245429372346,
    "I_sealed": 55.909083689349,
    "base": 56.828664559360675,
    "excess": -0.7342259024149009,
    "gap": 1.8391617400233429,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 77.03865189535385,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 75.46398238439761,
       "judge": 76.26895187870797,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 45.291235334713605,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 50.91484905804961,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.85808404482206,
       "hard": 54.85367186261314,
       "judge": 58.70222464767498,
       "standard": 25.384522571098287
      }
     },
     "sealed|choice": {
      "cc": 72.15078699038692,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 66.4027149321267,
       "judge": 72.06703910614526,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 42.779753431164394,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 24.271844660194162,
       "judge": 42.10526315789473,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 52.79671064649569,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.73482464128581,
       "hard": 48.652428202150055,
       "judge": 63.82621512662949,
       "standard": 41.571961817591195
      }
     }
    }
   },
   "jevbench_score": 21.274526640872896,
   "key": "jev-27b-autotrust",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "autotrust/JEV-27B@51740a8891c2a8baefd969237fd44187b3e3a115",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9452054794520548,
    "decisive_rate": 0.7870619946091644,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.89967561034447,
     "cost": 31.02571003473804,
     "intelligence": 56.828664559360675,
     "speed": 86.89503266505372
    },
    "board_release": "v1.6.43",
    "capability": 73.86417008485257,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.660744990021566,
      21.639609973811442
     ],
     "B": [
      20.50360595860232,
      22.019717921334916
     ],
     "C": [
      15.257925180346783,
      21.639609973811442
     ]
    },
    "jevbench_score": 21.291230595611577,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 88,
    "rank": 122,
    "ranks": {
     "A": 122,
     "B": 121,
     "C": 117,
     "capability": 49
    },
    "scores": {
     "A": 21.291230595611577,
     "B": 21.406663290369472,
     "C": 19.09999105447478
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3604971420194488,
     "p50_s_raw": 0.1052485710097244,
     "p95_s_adjusted": 0.5670154692983488,
     "p95_s_raw": 0.20850773464917438
    },
    "views": {
     "Balanced 33:33:33": 18.83248652432942,
     "Emphasis on Accuracy 60:20:20": 19.94400418862255,
     "Emphasis on Cost 20:20:60": 15.303706986099348,
     "Emphasis on Speed 20:60:20": 22.82314538634501,
     "Intelligence only": 21.88118814236141
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 138,
   "ranked": true,
   "ranks": {
    "A": 138,
    "B": 138,
    "C": 134,
    "capability": 60
   },
   "repo": "https://huggingface.co/autotrust/JEV-27B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.274526640872896,
    "B": 21.393152760173372,
    "C": 19.085006228461534
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.37615348198451104,
    "p50_s_raw": 0.11307674099225551,
    "p95_s_adjusted": 0.5995241465250728,
    "p95_s_raw": 0.22476207326253642
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.12231981093008,
      91.47472229481626
     ],
     "I": [
      51.54879559651137,
      61.562027595056925
     ],
     "capability": [
      69.671164690791,
      75.76898842923057
     ],
     "composite": {
      "A": [
       20.64501531235954,
       21.6223551289021
      ],
      "B": [
       20.4912109596719,
       22.005422726140193
      ],
      "C": [
       15.246303093563608,
       21.6223551289021
      ]
     },
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.89967561034447,
     "capability": 73.86417008485257,
     "intelligence": 56.828664559360675
    },
    "per_type": {
     "choice": {
      "accuracy": 0.796,
      "calibration": 91.08148167830316,
      "cc": 73.91131362314447,
      "cc_by_type": {
       "choice": 73.91131362314447
      },
      "cc_tiered": 74.59471944287039,
      "cc_tiered_open": 77.03865189535385,
      "cc_tiered_sealed": 72.15078699038692,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.91131362314447
     },
     "noul": {
      "accuracy": 0.736,
      "calibration": 90.93536952012767,
      "cc": 47.199999999999996,
      "cc_by_type": {
       "noul": 47.199999999999996
      },
      "cc_tiered": 44.035494382939,
      "cc_tiered_open": 45.291235334713605,
      "cc_tiered_sealed": 42.779753431164394,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 47.199999999999996
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 90.68217563260262,
      "cc": 51.866475511741164,
      "cc_by_type": {
       "score": 51.866475511741164
      },
      "cc_tiered": 51.85577985227265,
      "cc_tiered_open": 50.91484905804961,
      "cc_tiered_sealed": 52.79671064649569,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.866475511741164
     }
    },
    "raw_on_A": {
     "calibration": 88.71172832565286,
     "capability": 73.81510020130236,
     "intelligence": 58.91847207695186
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.815064013732968,
    "Emphasis on Accuracy 60:20:20": 19.932276352772433,
    "Emphasis on Cost 20:20:60": 15.296800679226827,
    "Emphasis on Speed 20:60:20": 22.777136206409697,
    "Intelligence only": 21.88118814236141
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 90.04368712409786,
    "cost": 30.341044859222123,
    "intelligence": 70.39456337588172,
    "speed": 87.62649746267333
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05156183229850985,
      "ece_score": 89.68763354029802,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 88.13304789226619,
      "score_before_time_limit": 89.68763354029802,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.16800000000000004,
      "brier": 0.06086846046546967,
      "ece": 0.03262322669949693,
      "n": 375,
      "score": 92.47828421039287,
      "score_before_time_limit": 93.47535466010062,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.992,
      "interval90_width": 0.2688888888888889,
      "n": 375,
      "nrps": 0.04123358842551135,
      "score": 89.51972926963455,
      "score_before_time_limit": 90.97533462361235,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.06962985955112075
     }
    },
    "score": 90.04368712409786
   },
   "capability": 80.21912524998979,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     20.730418962737154,
     21.33550691366303
    ],
    "B": [
     21.338683338124486,
     22.32081989164745
    ],
    "C": [
     20.730418962737154,
     21.33550691366303
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.71455888
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.20987715639810428
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "torchcast-decision-27b (Torchcast AI, Qwen3.8-27B fine-tune)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 71.86444089391547,
    "I_sealed": 68.92468585784798,
    "base": 70.39456337588172,
    "excess": 0.36636739362925175,
    "gap": 2.9397550360674956,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 82.0423850709834,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 83.01352626612143,
       "judge": 84.17930125247197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 62.8191856452726,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 73.91304347826086,
       "judge": 50.0,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 70.73175196549042,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 72.80894472531861,
       "hard": 75.13092960510187,
       "judge": 75.39366934082463,
       "standard": 53.90192424335207
      }
     },
     "sealed|choice": {
      "cc": 81.30477918983894,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 76.07466063348416,
       "judge": 85.1024208566108,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 56.67251541954846,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80.0,
       "hard": 39.80582524271845,
       "judge": 63.1578947368421,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 68.79676296415653,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 75.23124590065085,
       "hard": 60.91072527165571,
       "judge": 75.82659136309096,
       "standard": 70.80685428250935
      }
     }
    }
   },
   "jevbench_score": 21.137804661287618,
   "key": "torchcast-decision-27b",
   "last_measured_on": "2026-10-06",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-27b@08645834899cb8e00d370b77a3f4e0baa3241dd1",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.970873786407767,
    "decisive_rate": 0.8328840970350404,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.379440941337,
     "cost": 30.341044859222123,
     "intelligence": 71.75187578095783,
     "speed": 87.72436167205926
    },
    "board_release": "v1.6.43",
    "capability": 81.56565836114741,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.87995157412942,
      21.459724866005526
     ],
     "B": [
      21.530802573949977,
      22.491279110132325
     ],
     "C": [
      20.87995157412942,
      21.459724866005526
     ]
    },
    "jevbench_score": 21.273291266043532,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 89,
    "rank": 123,
    "ranks": {
     "A": 123,
     "B": 115,
     "C": 104,
     "capability": 19
    },
    "scores": {
     "A": 21.273291266043532,
     "B": 22.135897876506068,
     "C": 21.273291266043532
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3445720289950259,
     "p50_s_raw": 0.09728601449751295,
     "p95_s_adjusted": 0.4900989957066485,
     "p95_s_raw": 0.17004949785332427
    },
    "views": {
     "Balanced 33:33:33": 18.95010611000118,
     "Emphasis on Accuracy 60:20:20": 21.366887812526656,
     "Emphasis on Cost 20:20:60": 14.822679909386718,
     "Emphasis on Speed 20:60:20": 22.70411049156899,
     "Intelligence only": 26.42130811221146
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 139,
   "ranked": true,
   "ranks": {
    "A": 139,
    "B": 132,
    "C": 119,
    "capability": 29
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.137804661287618,
    "B": 21.947881456101502,
    "C": 21.137804661287618
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.34793927800492386,
    "p50_s_raw": 0.09896963900246192,
    "p95_s_adjusted": 0.49641717158956455,
    "p95_s_raw": 0.17320858579478227
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.79008978256473,
      90.87788667710647
     ],
     "I": [
      65.2932742596741,
      74.33171049634291
     ],
     "capability": [
      76.43449921363415,
      81.80366414311386
     ],
     "composite": {
      "A": [
       20.730418962737154,
       21.33550691366303
      ],
      "B": [
       21.338683338124486,
       22.32081989164745
      ],
      "C": [
       20.730418962737154,
       21.33550691366303
      ]
     },
     "seed": 120
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.04368712409786,
     "capability": 80.21912524998979,
     "intelligence": 70.39456337588172
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8586666666666667,
      "calibration": 88.13304789226619,
      "cc": 81.92548525525041,
      "cc_by_type": {
       "choice": 81.92548525525041
      },
      "cc_tiered": 81.67358213041118,
      "cc_tiered_open": 82.0423850709834,
      "cc_tiered_sealed": 81.30477918983894,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.92548525525041
     },
     "noul": {
      "accuracy": 0.8,
      "calibration": 92.47828421039287,
      "cc": 60.00000000000001,
      "cc_by_type": {
       "noul": 60.00000000000001
      },
      "cc_tiered": 59.74585053241053,
      "cc_tiered_open": 62.8191856452726,
      "cc_tiered_sealed": 56.67251541954846,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.00000000000001
     },
     "score": {
      "accuracy": 0.8133333333333334,
      "calibration": 89.51972926963455,
      "cc": 68.85226899393291,
      "cc_by_type": {
       "score": 68.85226899393291
      },
      "cc_tiered": 69.76425746482347,
      "cc_tiered_open": 70.73175196549042,
      "cc_tiered_sealed": 68.79676296415653,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.85226899393291
     }
    },
    "raw_on_A": {
     "calibration": 84.59019622902517,
     "capability": 79.48905505903151,
     "intelligence": 74.38791388903786
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.859052040669425,
    "Emphasis on Accuracy 60:20:20": 21.165739141316237,
    "Emphasis on Cost 20:20:60": 14.789168738726588,
    "Emphasis on Speed 20:60:20": 22.61850534998738,
    "Intelligence only": 25.921502791880584
   }
  },
  {
   "adapter_id": "images/nimble-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 69.88762704651819,
    "cost": 36.75037423027614,
    "intelligence": 43.14237010986392,
    "speed": 85.53966579201045
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.178988177870932,
      "ece_score": 64.2023644258136,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 64.2023644258136,
      "score_before_time_limit": 64.2023644258136,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2264150943396226,
      "brier": 0.14509070919724038,
      "ece": 0.1305339256295796,
      "n": 371,
      "score": 73.89321487408408,
      "score_before_time_limit": 73.89321487408408,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8401084010840109,
      "interval90_width": 0.26874435411020764,
      "n": 369,
      "nrps": 0.11567566172039558,
      "score": 71.56730183965689,
      "score_before_time_limit": 71.56730183965689,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.22648915074323328
     }
    },
    "score": 69.88762704651819
   },
   "capability": 56.514998578191054,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     15.028591067557413,
     26.025652117391928
    ],
    "B": [
     14.100620259268231,
     25.262854765602714
    ],
    "C": [
     10.43652157469265,
     18.073369525966612
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.1283278325123153
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble 9B (Bespoke Labs)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 46.96255933707536,
    "I_sealed": 39.32218088265248,
    "base": 43.14237010986392,
    "excess": 5.0669908119846365,
    "gap": 7.64037845442288,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 59.86495466898259,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 49.040578798364265,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 38.9399585921325,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 47.82608695652173,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 42.08276475011101,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.18206312365201,
       "hard": 44.02495901721014,
       "judge": 36.708929948504554,
       "standard": 43.70947923155191
      }
     },
     "sealed|choice": {
      "cc": 57.35467556665829,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 60.0916414160077,
       "hard": 46.5497737556561,
       "judge": 60.89385474860336,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 27.490153799650226,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52.0,
       "hard": 4.854368932038833,
       "judge": 36.8421052631579,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 33.12171328164895,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 49.98365384887967,
       "hard": 27.146627400052527,
       "judge": 32.82017491999325,
       "standard": 37.093222303709986
      }
     }
    }
   },
   "jevbench_score": 21.06112507504102,
   "key": "nimble-9b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (weights); repository without a licence file as of 19 Sep",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.867595818815331,
    "decisive_rate": 0.7735849056603774,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 69.88762704651819,
     "cost": 36.75037423027614,
     "intelligence": 43.14237010986392,
     "speed": 86.11327010751721
    },
    "board_release": "v1.6.43",
    "capability": 56.514998578191054,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      15.043066969876142,
      26.052957246349575
     ],
     "B": [
      14.110886674014932,
      25.283661649629934
     ],
     "C": [
      10.446574284636212,
      18.092331421076096
     ]
    },
    "jevbench_score": 21.082616704957896,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 90,
    "rank": 124,
    "ranks": {
     "A": 124,
     "B": 127,
     "C": 137,
     "capability": 157
    },
    "scores": {
     "A": 21.082616704957896,
     "B": 20.213517776427228,
     "C": 14.64070604510965
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.43987150778993966,
     "p50_s_raw": 0.14493575389496982,
     "p95_s_adjusted": 0.5563488021492958,
     "p95_s_raw": 0.2031744010746479
    },
    "views": {
     "Balanced 33:33:33": 19.46099526058868,
     "Emphasis on Accuracy 60:20:20": 18.55883910946502,
     "Emphasis on Cost 20:20:60": 17.273520313490142,
     "Emphasis on Speed 20:60:20": 23.596180145578664,
     "Intelligence only": 17.35223877358268
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 140,
   "ranked": true,
   "ranks": {
    "A": 140,
    "B": 147,
    "C": 155,
    "capability": 174
   },
   "repo": "https://github.com/bespokelabsai/nimble",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.06112507504102,
    "B": 20.19770903586119,
    "C": 14.62578130211182
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.46431010039523246,
    "p50_s_raw": 0.15715505019761622,
    "p95_s_adjusted": 0.601485676318407,
    "p95_s_raw": 0.2257428381592035
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "bespokelabs/Bespoke-Nimble-9B (LoRA, adapter unchanged since 93ec5d6), merged into Qwen/Qwen3.5-9B@c202236 with the author's PEFT safe-merge",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      65.1956012319158,
      73.91457472170808
     ],
     "I": [
      37.37380617272022,
      47.11791102705384
     ],
     "capability": [
      52.152502588110366,
      59.801663230716926
     ],
     "composite": {
      "A": [
       15.028591067557413,
       26.025652117391928
      ],
      "B": [
       14.100620259268231,
       25.262854765602714
      ],
      "C": [
       10.43652157469265,
       18.073369525966612
      ]
     },
     "seed": 71
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 69.88762704651819,
     "capability": 56.514998578191054,
     "intelligence": 43.14237010986392
    },
    "per_type": {
     "choice": {
      "accuracy": 0.668,
      "calibration": 64.2023644258136,
      "cc": 57.541941778842954,
      "cc_by_type": {
       "choice": 57.541941778842954
      },
      "cc_tiered": 58.60981511782044,
      "cc_tiered_open": 59.86495466898259,
      "cc_tiered_sealed": 57.35467556665829,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.541941778842954
     },
     "noul": {
      "accuracy": 0.664,
      "calibration": 73.89321487408408,
      "cc": 32.800000000000004,
      "cc_by_type": {
       "noul": 32.800000000000004
      },
      "cc_tiered": 33.215056195891364,
      "cc_tiered_open": 38.9399585921325,
      "cc_tiered_sealed": 27.490153799650226,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.800000000000004
     },
     "score": {
      "accuracy": 0.536,
      "calibration": 71.56730183965689,
      "cc": 35.587315474154515,
      "cc_by_type": {
       "score": 35.587315474154515
      },
      "cc_tiered": 37.60223901587998,
      "cc_tiered_open": 42.08276475011101,
      "cc_tiered_sealed": 33.12171328164895,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.587315474154515
     }
    },
    "raw_on_A": {
     "calibration": 72.02241547589321,
     "capability": 59.4874434751425,
     "intelligence": 46.95247147439178
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.436584180238565,
    "Emphasis on Accuracy 60:20:20": 18.545511787500214,
    "Emphasis on Cost 20:20:60": 17.26197449871947,
    "Emphasis on Speed 20:60:20": 23.531678747407696,
    "Intelligence only": 17.35223877358268
   }
  },
  {
   "adapter_id": "images/reflex-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kshetrajna12",
   "axes": {
    "calibration": 86.9025081232179,
    "cost": 63.142712976263866,
    "intelligence": 30.80973256512164,
    "speed": 68.75878067126158
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.068294434375,
      "ece_score": 86.341113125,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 86.11087015666666,
      "score_before_time_limit": 86.341113125,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0.41866666666666663,
      "brier": 0.14083422852570657,
      "ece": 0.05314781866666669,
      "n": 375,
      "score": 89.37043626666666,
      "score_before_time_limit": 89.37043626666666,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9866666666666667,
      "interval90_width": 0.7284444444444443,
      "n": 375,
      "nrps": 0.1108956250735923,
      "score": 85.22621794632039,
      "score_before_time_limit": 85.22621794632039,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09229000799999995
     }
    },
    "score": 86.9025081232179
   },
   "capability": 58.85612034416977,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     14.609726845110593,
     29.096277415450384
    ],
    "B": [
     12.443877514437913,
     26.040177253959957
    ],
    "C": [
     10.145643642437912,
     20.205748205173876
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.016926853448275863
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "reflex 4B (kshetrajna12)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 33.07061549561215,
    "I_sealed": 28.548849634631125,
    "base": 30.80973256512164,
    "excess": 1.9483782185427834,
    "gap": 4.521765860981027,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 54.32138182784878,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 49.040578798364265,
       "judge": 48.58272907053395,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 10.45962732919255,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 34.430837329795125,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.20111818181819,
       "hard": 24.787801851851842,
       "judge": 33.86929491525422,
       "standard": 45.67408148148149
      }
     },
     "sealed|choice": {
      "cc": 53.57907452177943,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 50.113122171945705,
       "judge": 50.65176908752329,
       "standard": 58.9850966867841
      }
     },
     "sealed|noul": {
      "cc": 1.7255356358899414,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -14.563106796116509,
       "judge": 5.263157894736836,
       "standard": 9.859154929577475
      }
     },
     "sealed|score": {
      "cc": 30.341938746224002,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 54.91348368794326,
       "hard": 26.727804411764712,
       "judge": 23.787971856287427,
       "standard": 35.11538527918782
      }
     }
    }
   },
   "jevbench_score": 20.428812955794445,
   "key": "reflex-4b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (code, adapter); Apache-2.0 (base)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9174311926605505,
    "decisive_rate": 0.5813333333333334,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.97925577932901,
     "cost": 63.142712976263866,
     "intelligence": 30.935558296483705,
     "speed": 68.7900017304639
    },
    "board_release": "v1.6.43",
    "capability": 58.95740703790636,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      14.765436275328822,
      29.209404384423753
     ],
     "B": [
      12.581433816044255,
      26.114502689493058
     ],
     "C": [
      10.253775191200571,
      20.28430860029427
     ]
    },
    "jevbench_score": 20.637311182702398,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 92,
    "rank": 126,
    "ranks": {
     "A": 126,
     "B": 137,
     "C": 138,
     "capability": 138
    },
    "scores": {
     "A": 20.637311182702398,
     "B": 17.968342404009285,
     "C": 14.33146609909889
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 2.865421538101509,
     "p50_s_raw": 1.3577107690507546,
     "p95_s_adjusted": 4.611171829840169,
     "p95_s_raw": 2.230585914920084
    },
    "views": {
     "Balanced 33:33:33": 18.316133226478374,
     "Emphasis on Accuracy 60:20:20": 15.029593546249647,
     "Emphasis on Cost 20:20:60": 20.28126457022741,
     "Emphasis on Speed 20:60:20": 20.855909642703498,
     "Intelligence only": 11.842240202134587
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 146,
   "ranked": true,
   "ranks": {
    "A": 146,
    "B": 155,
    "C": 156,
    "capability": 156
   },
   "repo": "https://github.com/kshetrajna12/reflex",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.428812955794445,
    "B": 17.775617173764044,
    "C": 14.186675663746145
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.8740736179519444,
    "p50_s_raw": 1.3620368089759722,
    "p95_s_adjusted": 4.630459016701207,
    "p95_s_raw": 2.240229508350603
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B + kshetrajna12/reflex-qwen3.5-4b-lora",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.4557796549836,
      87.34421355395163
     ],
     "I": [
      26.928642054525277,
      35.778887054174305
     ],
     "capability": [
      54.73340669244083,
      60.40734224969147
     ],
     "composite": {
      "A": [
       14.609726845110593,
       29.096277415450384
      ],
      "B": [
       12.443877514437913,
       26.040177253959957
      ],
      "C": [
       10.145643642437912,
       20.205748205173876
      ]
     },
     "seed": 98
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.9025081232179,
     "capability": 58.85612034416977,
     "intelligence": 30.80973256512164
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6466666666666666,
      "calibration": 86.11087015666666,
      "cc": 54.813713138126026,
      "cc_by_type": {
       "choice": 54.813713138126026
      },
      "cc_tiered": 53.95022817481411,
      "cc_tiered_open": 54.32138182784878,
      "cc_tiered_sealed": 53.57907452177943,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.813713138126026
     },
     "noul": {
      "accuracy": 0.5333333333333333,
      "calibration": 89.37043626666666,
      "cc": 6.666666666666665,
      "cc_by_type": {
       "noul": 6.666666666666665
      },
      "cc_tiered": 6.092581482541245,
      "cc_tiered_open": 10.45962732919255,
      "cc_tiered_sealed": 1.7255356358899414,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 6.666666666666665
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 85.22621794632039,
      "cc": 32.60462780701755,
      "cc_by_type": {
       "score": 32.60462780701755
      },
      "cc_tiered": 32.386388038009564,
      "cc_tiered_open": 34.430837329795125,
      "cc_tiered_sealed": 30.341938746224002,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.60462780701755
     }
    },
    "raw_on_A": {
     "calibration": 81.34463076766183,
     "capability": 58.29083157222991,
     "intelligence": 35.237032376798
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.002666666666666706,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.127364017655474,
    "Emphasis on Accuracy 60:20:20": 14.860595774408743,
    "Emphasis on Cost 20:20:60": 20.087114192687814,
    "Emphasis on Speed 20:60:20": 20.652433547341495,
    "Intelligence only": 11.698327542016717
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Perplexity",
   "axes": {
    "calibration": 89.82871864473867,
    "cost": 29.886617009940878,
    "intelligence": 75.59095372376527,
    "speed": 86.35967790128073
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06981468503472926,
      "ece_score": 86.03706299305415,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.9223469090634,
      "score_before_time_limit": 86.03706299305415,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.0889487870619946,
      "brier": 0.062186325211303205,
      "ece": 0.051085406690107325,
      "n": 371,
      "score": 89.78291866197854,
      "score_before_time_limit": 89.78291866197854,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9566395663956639,
      "interval90_width": 0.165085817524842,
      "n": 369,
      "nrps": 0.04050151348288236,
      "score": 93.78089036317405,
      "score_before_time_limit": 93.78089036317405,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04194033962681827
     }
    },
    "score": 89.82871864473867
   },
   "capability": 82.70983618425197,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     20.245453170923156,
     20.77609725321893
    ],
    "B": [
     21.091992895392327,
     21.92824201074229
    ],
    "C": [
     20.245453170923156,
     20.77609725321893
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.21399420000000002
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.21732654028436021
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Perplexity Decider v1.1 27B (Qwen3.8-27B, noncausal decision readout)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 78.11444356279019,
    "I_sealed": 73.06746388474036,
    "base": 75.59095372376527,
    "excess": 2.473592035611579,
    "gap": 5.046979678049823,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 86.99770517622916,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 84.90091223655237,
       "judge": 88.13447593935398,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 77.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 70.26833082615104,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.42506419505849,
       "hard": 75.99276310558226,
       "judge": 80.94030641691248,
       "standard": 46.23313619669274
      }
     },
     "sealed|choice": {
      "cc": 80.03085922219567,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 71.49321266968325,
       "judge": 86.96461824953444,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 66.96797052113396,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 53.398058252427184,
       "judge": 71.05263157894737,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 72.20356191089144,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 73.73143188188055,
       "hard": 66.84626737629874,
       "judge": 80.0022436490447,
       "standard": 70.4561933873524
      }
     }
    }
   },
   "jevbench_score": 20.5925378813709,
   "key": "pplx-decider-v1-1-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "perplexity-ai/pplx-decider-v1.1-27b@3b45dead91dfa6d95aad6b95764a606fab2bf7a6",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9497041420118343,
    "decisive_rate": 0.9110512129380054,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.86695733940225,
     "cost": 29.886617009940878,
     "intelligence": 75.71677945512734,
     "speed": 86.79279341461623
    },
    "board_release": "v1.6.43",
    "capability": 82.79186839726479,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.265432194933428,
      20.803744956997086
     ],
     "B": [
      21.111865782219617,
      21.95821416195702
     ],
     "C": [
      20.265432194933428,
      20.803744956997086
     ]
    },
    "jevbench_score": 20.617642759667973,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 93,
    "rank": 127,
    "ranks": {
     "A": 127,
     "B": 117,
     "C": 109,
     "capability": 12
    },
    "scores": {
     "A": 20.617642759667973,
     "B": 21.647466573268815,
     "C": 20.617642759667973
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.39472174100810664,
     "p50_s_raw": 0.12236087050405331,
     "p95_s_adjusted": 0.5301876544195693,
     "p95_s_raw": 0.19009382720978463
    },
    "views": {
     "Balanced 33:33:33": 18.420307065234873,
     "Emphasis on Accuracy 60:20:20": 21.115368174311886,
     "Emphasis on Cost 20:20:60": 14.279000034452622,
     "Emphasis on Speed 20:60:20": 21.991580334127057,
     "Intelligence only": 27.052390084345777
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 143,
   "ranked": true,
   "ranks": {
    "A": 143,
    "B": 134,
    "C": 125,
    "capability": 14
   },
   "repo": "https://huggingface.co/perplexity-ai/pplx-decider-v1.1-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.5925378813709,
    "B": 21.619568400558805,
    "C": 20.5925378813709
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4121504910348449,
    "p50_s_raw": 0.13107524551742245,
    "p95_s_adjusted": 0.5610174738016211,
    "p95_s_raw": 0.2055087369008106
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.8241744764109,
      91.28720890209534
     ],
     "I": [
      70.85113772743928,
      78.95089291005449
     ],
     "capability": [
      79.1028452853472,
      84.63982686991523
     ],
     "composite": {
      "A": [
       20.245453170923156,
       20.77609725321893
      ],
      "B": [
       21.091992895392327,
       21.92824201074229
      ],
      "C": [
       20.245453170923156,
       20.77609725321893
      ]
     },
     "seed": 95
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.82871864473867,
     "capability": 82.70983618425197,
     "intelligence": 75.59095372376527
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 85.9223469090634,
      "cc": 81.41394238511599,
      "cc_by_type": {
       "choice": 81.41394238511599
      },
      "cc_tiered": 83.51428219921242,
      "cc_tiered_open": 86.99770517622916,
      "cc_tiered_sealed": 80.03085922219567,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.41394238511599
     },
     "noul": {
      "accuracy": 0.856,
      "calibration": 89.78291866197854,
      "cc": 71.2,
      "cc_by_type": {
       "noul": 71.2
      },
      "cc_tiered": 72.02263260356216,
      "cc_tiered_open": 77.07729468599035,
      "cc_tiered_sealed": 66.96797052113396,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 71.2
     },
     "score": {
      "accuracy": 0.8133333333333334,
      "calibration": 93.78089036317405,
      "cc": 71.25760707133534,
      "cc_by_type": {
       "score": 71.25760707133534
      },
      "cc_tiered": 71.23594636852124,
      "cc_tiered_open": 70.26833082615104,
      "cc_tiered_sealed": 72.20356191089144,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 71.25760707133534
     }
    },
    "raw_on_A": {
     "calibration": 89.31143519070639,
     "capability": 82.8788987724678,
     "intelligence": 76.44636235422921
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.395090031605847,
    "Emphasis on Accuracy 60:20:20": 21.08453088019151,
    "Emphasis on Cost 20:20:60": 14.26990163000141,
    "Emphasis on Speed 20:60:20": 21.93882456548038,
    "Intelligence only": 27.007434569967725
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "theunnecessarythings",
   "axes": {
    "calibration": 50.5645403785324,
    "cost": 33.57969498589053,
    "intelligence": 47.93439876939806,
    "speed": 85.09218748397188
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14830087112828266,
      "ece_score": 70.33982577434347,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 70.33982577434347,
      "score_before_time_limit": 70.33982577434347,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.05660377358490565,
      "brier": 0.09551749322805621,
      "ece": 0.09323102319373136,
      "n": 371,
      "score": 81.35379536125373,
      "score_before_time_limit": 81.35379536125373,
      "time_limit_failed_share": 0.0
     }
    },
    "score": 50.5645403785324
   },
   "capability": 49.24946957396523,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     16.710624359003894,
     22.523467221016634
    ],
    "B": [
     16.48176787806923,
     22.63582853526852
    ],
    "C": [
     11.604600249308263,
     16.706994443865153
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.16117564
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.16368548408937036
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JADE (Qwen3.8-27B LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 47.17172129671271,
    "I_sealed": 48.6970762420834,
    "base": 47.93439876939806,
    "excess": -4.09874258780893,
    "gap": -1.5253549453706867,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 80.09487403506567,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 73.57659641396666,
       "judge": 80.22412656558997,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 61.42028985507247,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -163.63636363636363,
       "hard": -161.1111111111111,
       "judge": -213.55932203389827,
       "standard": -355.55555555555554
      }
     },
     "sealed|choice": {
      "cc": 75.66415221506249,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 70.47511312217195,
       "judge": 76.72253258845438,
       "standard": 82.26382559428504
      }
     },
     "sealed|noul": {
      "cc": 70.42707651118772,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80.0,
       "hard": 57.28155339805825,
       "judge": 76.3157894736842,
       "standard": 83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -182.97872340425533,
       "hard": -183.82352941176472,
       "judge": -204.79041916167668,
       "standard": -228.42639593908638
      }
     }
    }
   },
   "jevbench_score": 20.179601004732195,
   "key": "jade",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "theunnecessarythings/JADE@8e9f9a3aa4293190fcc4a6ff17ff5f3251f4c270",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9142857142857143,
    "decisive_rate": 0.9433962264150944,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 50.5645403785324,
     "cost": 33.57969498589053,
     "intelligence": 47.93439876939806,
     "speed": 85.53222225487117
    },
    "board_release": "v1.6.43",
    "capability": 49.24946957396523,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      16.72243182039576,
      22.540480675102366
     ],
     "B": [
      16.49103923273388,
      22.64957340042747
     ],
     "C": [
      11.612799875274835,
      16.71953364433383
     ]
    },
    "jevbench_score": 20.194459825016597,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 96,
    "rank": 131,
    "ranks": {
     "A": 131,
     "B": 129,
     "C": 139,
     "capability": 177
    },
    "scores": {
     "A": 20.194459825016597,
     "B": 20.12888689696233,
     "C": 14.023930434039306
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4045139710127842,
     "p50_s_raw": 0.12725698550639208,
     "p95_s_adjusted": 0.6915829036966897,
     "p95_s_raw": 0.27079145184834486
    },
    "views": {
     "Balanced 33:33:33": 19.951224387926274,
     "Emphasis on Accuracy 60:20:20": 19.918976230985706,
     "Emphasis on Cost 20:20:60": 17.004313740085063,
     "Emphasis on Speed 20:60:20": 24.181035896264405,
     "Intelligence only": 19.870798988937533
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 149,
   "ranked": true,
   "ranks": {
    "A": 149,
    "B": 148,
    "C": 157,
    "capability": 197
   },
   "repo": "https://huggingface.co/theunnecessarythings/JADE",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.179601004732195,
    "B": 20.117075150970894,
    "C": 14.013611808841805
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.42184902198496277,
    "p50_s_raw": 0.13592451099248137,
    "p95_s_adjusted": 0.7338785682804883,
    "p95_s_raw": 0.2919392841402441
   },
   "status": {
    "answered_ok": 1108,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      46.63888535444405,
      53.10214733728139
     ],
     "I": [
      44.307422428017674,
      51.72167160495215
     ],
     "capability": [
      45.932872400615175,
      51.86244666467468
     ],
     "composite": {
      "A": [
       16.710624359003894,
       22.523467221016634
      ],
      "B": [
       16.48176787806923,
       22.63582853526852
      ],
      "C": [
       11.604600249308263,
       16.706994443865153
      ]
     },
     "seed": 47
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 50.5645403785324,
     "capability": 49.24946957396523,
     "intelligence": 47.93439876939806
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8186666666666667,
      "calibration": 70.33982577434347,
      "cc": 76.8100565539062,
      "cc_by_type": {
       "choice": 76.8100565539062
      },
      "cc_tiered": 77.87951312506408,
      "cc_tiered_open": 80.09487403506567,
      "cc_tiered_sealed": 75.66415221506249,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.8100565539062
     },
     "noul": {
      "accuracy": 0.8533333333333334,
      "calibration": 81.35379536125373,
      "cc": 70.66666666666667,
      "cc_by_type": {
       "noul": 70.66666666666667
      },
      "cc_tiered": 65.9236831831301,
      "cc_tiered_open": 61.42028985507247,
      "cc_tiered_sealed": 70.42707651118772,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 70.66666666666667
     },
     "score": {
      "accuracy": 0.0,
      "calibration": null,
      "cc": -197.36842105263145,
      "cc_by_type": {
       "score": -197.36842105263145
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 51.05198021321377,
     "capability": 50.51094734813176,
     "intelligence": 49.96991448304975
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 1.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.931891518461057,
    "Emphasis on Accuracy 60:20:20": 19.907409483244074,
    "Emphasis on Cost 20:20:60": 16.995883647201563,
    "Emphasis on Speed 20:60:20": 24.129975692688248,
    "Intelligence only": 19.870798988937533
   }
  },
  {
   "adapter_id": "images/typecastlm/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mikhail Gribov",
   "axes": {
    "calibration": 83.29951051808085,
    "cost": 63.95900966831593,
    "intelligence": 29.78900656204624,
    "speed": 92.66059621716312
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14894691278376873,
      "ece_score": 70.21061744324626,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 70.21061744324626,
      "score_before_time_limit": 70.21061744324626,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.42666666666666664,
      "brier": 0.1443460310902931,
      "ece": 0.03743872703084413,
      "n": 375,
      "score": 92.51225459383117,
      "score_before_time_limit": 92.51225459383117,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9973333333333333,
      "interval90_width": 0.8611111111111112,
      "n": 375,
      "nrps": 0.11438335298438909,
      "score": 87.17565951716512,
      "score_before_time_limit": 87.17565951716512,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07105172833615442
     }
    },
    "score": 83.29951051808085
   },
   "capability": 56.544258540063545,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     12.878604696443892,
     26.09986756824515
    ],
    "B": [
     10.665475260096787,
     22.721969740865802
    ],
    "C": [
     8.94347548364159,
     18.124908033503576
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01589887315270936
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "typecastlm (Mikhail Gribov, Qwen3.5-4B computed head)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 34.211044348094305,
    "I_sealed": 25.366968775998174,
    "base": 29.78900656204624,
    "excess": 6.270687929657887,
    "gap": 8.84407557209613,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 53.948175356316256,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 45.26580685750237,
       "judge": 52.537903757415954,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 19.30296756383713,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": 39.13043478260869,
       "judge": 0.0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 29.381990124129523,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.87815307647144,
       "hard": 32.63833455597018,
       "judge": 28.21824515037382,
       "standard": 9.366837244910798
      }
     },
     "sealed|choice": {
      "cc": 46.35542091610937,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 37.89592760180995,
       "judge": 51.58286778398511,
       "standard": 47.89998768321223
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 19.999999999999996,
       "hard": -22.330097087378643,
       "judge": 2.6315789473684292,
       "standard": 15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 29.745485411885156,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.652319160951166,
       "hard": 25.940906558427802,
       "judge": 26.848256959038974,
       "standard": 30.247068923536112
      }
     }
    }
   },
   "jevbench_score": 19.71928370373473,
   "key": "typecastlm",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (package and weights)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9023255813953488,
    "decisive_rate": 0.5733333333333334,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.29951051808085,
     "cost": 63.95900966831593,
     "intelligence": 29.78900656204624,
     "speed": 92.76354061232337
    },
    "board_release": "v1.6.43",
    "capability": 56.544258540063545,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 76,
    "capability_rank": 102,
    "composite_ci95": {
     "A": [
      12.880576558614674,
      26.104438970675695
     ],
     "B": [
      10.666557109719218,
      22.72472546215454
     ],
     "C": [
      8.944844832371304,
      18.12808261852479
     ]
    },
    "jevbench_score": 19.72256430004839,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 98,
    "rank": 133,
    "ranks": {
     "A": 133,
     "B": 142,
     "C": 140,
     "capability": 156
    },
    "scores": {
     "A": 19.72256430004839,
     "B": 16.813084809424783,
     "C": 13.69622520836694
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20903298105113208,
     "p50_s_raw": 0.02951649052556604,
     "p95_s_adjusted": 0.25318100801086985,
     "p95_s_raw": 0.051590504005434926
    },
    "views": {
     "Balanced 33:33:33": 17.752271764169233,
     "Emphasis on Accuracy 60:20:20": 13.960999831173599,
     "Emphasis on Cost 20:20:60": 19.448558330257274,
     "Emphasis on Speed 20:60:20": 21.764369978155518,
     "Intelligence only": 10.57372598609905
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 151,
   "ranked": true,
   "ranks": {
    "A": 151,
    "B": 160,
    "C": 158,
    "capability": 173
   },
   "repo": "https://github.com/mihail-gribov/typecastlm",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 19.71928370373473,
    "B": 16.81117744446691,
    "C": 13.693947016482454
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21090144504560157,
    "p50_s_raw": 0.03045072252280079,
    "p95_s_adjusted": 0.25695721871452404,
    "p95_s_raw": 0.05347860935726202
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.58498026450093,
      85.01640087455884
     ],
     "I": [
      25.093945135273305,
      33.49496683838513
     ],
     "capability": [
      52.490530265893305,
      58.297640238106396
     ],
     "composite": {
      "A": [
       12.878604696443892,
       26.09986756824515
      ],
      "B": [
       10.665475260096787,
       22.721969740865802
      ],
      "C": [
       8.94347548364159,
       18.124908033503576
      ]
     },
     "seed": 107
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.29951051808085,
     "capability": 56.544258540063545,
     "intelligence": 29.78900656204624
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5933333333333334,
      "calibration": 70.21061744324626,
      "cc": 47.99314153633374,
      "cc_by_type": {
       "choice": 47.99314153633374
      },
      "cc_tiered": 50.15179813621281,
      "cc_tiered_open": 53.948175356316256,
      "cc_tiered_sealed": 46.35542091610937,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 47.99314153633374
     },
     "noul": {
      "accuracy": 0.5173333333333333,
      "calibration": 92.51225459383117,
      "cc": 3.4666666666666623,
      "cc_by_type": {
       "noul": 3.4666666666666623
      },
      "cc_tiered": 9.651483781918564,
      "cc_tiered_open": 19.30296756383713,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 3.4666666666666623
     },
     "score": {
      "accuracy": 0.48533333333333334,
      "calibration": 87.17565951716512,
      "cc": 31.463946158492806,
      "cc_by_type": {
       "score": 31.463946158492806
      },
      "cc_tiered": 29.56373776800734,
      "cc_tiered_open": 29.381990124129523,
      "cc_tiered_sealed": 29.745485411885156,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 31.463946158492806
     }
    },
    "raw_on_A": {
     "calibration": 79.3986887571586,
     "capability": 55.69880635159235,
     "intelligence": 31.998923946026117
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 17.748728054070742,
    "Emphasis on Accuracy 60:20:20": 13.959684665716393,
    "Emphasis on Cost 20:20:60": 19.446006180840094,
    "Emphasis on Speed 20:60:20": 21.754784574521864,
    "Intelligence only": 10.57372598609905
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Frontier Infra",
   "axes": {
    "calibration": 83.00593897276424,
    "cost": 29.935481526523958,
    "intelligence": 57.4596907179922,
    "speed": 85.07834149803253
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13434975244299677,
      "ece_score": 73.13004951140066,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 73.03254277871879,
      "score_before_time_limit": 73.13004951140066,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.26415094339622647,
      "brier": 0.06648894231173322,
      "ece": 0.08077593261455521,
      "n": 371,
      "score": 83.84481347708896,
      "score_before_time_limit": 83.84481347708896,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.981029810298103,
      "interval90_width": 0.36359530261969286,
      "n": 369,
      "nrps": 0.05869267834921634,
      "score": 92.14046066248498,
      "score_before_time_limit": 92.14046066248498,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.049249054200542
     }
    },
    "score": 83.00593897276424
   },
   "capability": 70.23281484537821,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     18.68928848729855,
     19.491338311704588
    ],
    "B": [
     18.668502978762927,
     19.984737466857364
    ],
    "C": [
     13.91237167979055,
     19.491338311704588
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.21319311999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.2165129857819905
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jebadiah 27B (Frontier Infra, Qwen3.8-27B LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.25677458890849,
    "I_sealed": 57.66260684707591,
    "base": 57.4596907179922,
    "excess": -2.9792199006056586,
    "gap": -0.40583225816741475,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 73.88416658400591,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 71.68921044353571,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 37.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 60.51762716201564,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.48122121212121,
       "hard": 54.84767685185186,
       "judge": 70.97751186440678,
       "standard": 51.685903703703694
      }
     },
     "sealed|choice": {
      "cc": 76.61944220069019,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 67.42081447963801,
       "judge": 83.24022346368716,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 39.64965634378576,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 18.446601941747566,
       "judge": 42.10526315789473,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 56.71872199675177,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.98117588652481,
       "hard": 44.34649191176467,
       "judge": 60.52794730538922,
       "standard": 69.61811725888323
      }
     }
    }
   },
   "jevbench_score": 19.217465890862947,
   "key": "jebadiah-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "frontier-infra/jebadiah-27b@4d3ed81640bbc015b47976bedae67212a5fb39cd",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9816849816849816,
    "decisive_rate": 0.7358490566037735,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.03844121699153,
     "cost": 29.935481526523958,
     "intelligence": 57.58551644935426,
     "speed": 85.83700487110255
    },
    "board_release": "v1.6.43",
    "capability": 70.3119788331729,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      18.744465890345797,
      19.52946206149045
     ],
     "B": [
      18.712637254320782,
      20.028302946408985
     ],
     "C": [
      13.997484201611632,
      19.52946206149045
     ]
    },
    "jevbench_score": 19.255307672764186,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 99,
    "rank": 134,
    "ranks": {
     "A": 134,
     "B": 131,
     "C": 124,
     "capability": 77
    },
    "scores": {
     "A": 19.255307672764186,
     "B": 19.517487118738075,
     "C": 17.736768346068793
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4285312689666171,
     "p50_s_raw": 0.13926563448330853,
     "p95_s_adjusted": 0.6085790472658118,
     "p95_s_raw": 0.2292895236329059
    },
    "views": {
     "Balanced 33:33:33": 17.22762964959432,
     "Emphasis on Accuracy 60:20:20": 18.448138764634205,
     "Emphasis on Cost 20:20:60": 13.86870216120643,
     "Emphasis on Speed 20:60:20": 20.908227606199645,
     "Intelligence only": 20.641713893950236
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 152,
   "ranked": true,
   "ranks": {
    "A": 152,
    "B": 149,
    "C": 142,
    "capability": 91
   },
   "repo": "https://huggingface.co/frontier-infra/jebadiah-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 19.217465890862947,
    "B": 19.478318832047695,
    "C": 17.624637213319602
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.463657653023256,
    "p50_s_raw": 0.156828826511628,
    "p95_s_adjusted": 0.6698358964873478,
    "p95_s_raw": 0.25991794824367387
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.4964804722844,
      84.29963743954265
     ],
     "I": [
      51.78610895497503,
      62.258853035805195
     ],
     "capability": [
      66.56436743169048,
      72.02289175876106
     ],
     "composite": {
      "A": [
       18.68928848729855,
       19.491338311704588
      ],
      "B": [
       18.668502978762927,
       19.984737466857364
      ],
      "C": [
       13.91237167979055,
       19.491338311704588
      ]
     },
     "seed": 49
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.00593897276424,
     "capability": 70.23281484537821,
     "intelligence": 57.4596907179922
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8133333333333334,
      "calibration": 73.03254277871879,
      "cc": 76.12799939372697,
      "cc_by_type": {
       "choice": 76.12799939372697
      },
      "cc_tiered": 75.25180439234805,
      "cc_tiered_open": 73.88416658400591,
      "cc_tiered_sealed": 76.61944220069019,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.12799939372697
     },
     "noul": {
      "accuracy": 0.7146666666666667,
      "calibration": 83.84481347708896,
      "cc": 42.93333333333334,
      "cc_by_type": {
       "noul": 42.93333333333334
      },
      "cc_tiered": 38.50909318224484,
      "cc_tiered_open": 37.36853002070393,
      "cc_tiered_sealed": 39.64965634378576,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 42.93333333333334
     },
     "score": {
      "accuracy": 0.6853333333333333,
      "calibration": 92.14046066248498,
      "cc": 57.405502456140376,
      "cc_by_type": {
       "score": 57.405502456140376
      },
      "cc_tiered": 58.618174579383705,
      "cc_tiered_open": 60.51762716201564,
      "cc_tiered_sealed": 56.71872199675177,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.405502456140376
     }
    },
    "raw_on_A": {
     "calibration": 80.08029667613602,
     "capability": 71.54861127321053,
     "intelligence": 63.01692587028502
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 17.188551643683,
    "Emphasis on Accuracy 60:20:20": 18.40684163881129,
    "Emphasis on Cost 20:20:60": 13.853489197602109,
    "Emphasis on Speed 20:60:20": 20.823282240360804,
    "Intelligence only": 20.596611255172014
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router",
   "axes": {
    "calibration": 82.20136692745442,
    "cost": 29.632311028770005,
    "intelligence": 60.4093162408047,
    "speed": 83.34942176442772
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10003484913300223,
      "ece_score": 79.99303017339956,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 79.99303017339956,
      "score_before_time_limit": 79.99303017339956,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2668463611859838,
      "brier": 0.06657517963022924,
      "ece": 0.0795050319911621,
      "n": 371,
      "score": 84.09899360176757,
      "score_before_time_limit": 84.09899360176757,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.994579945799458,
      "interval90_width": 0.5117434507678408,
      "n": 369,
      "nrps": 0.05676847122486311,
      "score": 82.51207700719613,
      "score_before_time_limit": 82.51207700719613,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.14649499431560725
     }
    },
    "score": 82.20136692745442
   },
   "capability": 71.30534158412956,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     18.436305597721148,
     19.058075982834907
    ],
    "B": [
     18.65398466828145,
     19.650592634580573
    ],
    "C": [
     15.730863308763542,
     19.058075982834907
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.21821211999999998
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.22161014218009478
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 2.0 Vega 27B (vLLM-SR, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 63.49716698880913,
    "I_sealed": 57.321465492800264,
    "base": 60.4093162408047,
    "excess": 3.6023138535706187,
    "gap": 6.1757014960088625,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 83.11168946691484,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 80.22412656558997,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 48.59696342305038,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 73.91304347826086,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 58.78284807646216,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.69543007415754,
       "hard": 61.18205862546275,
       "judge": 57.862992432344406,
       "standard": 53.407919445789865
      }
     },
     "sealed|choice": {
      "cc": 75.37523860459501,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 71.49321266968325,
       "judge": 77.6536312849162,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 40.19307217860048,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": 26.21359223300972,
       "judge": 42.10526315789473,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 56.396085695205315,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.340074967907654,
       "hard": 48.314713299858624,
       "judge": 65.93509972749679,
       "standard": 55.2783148011103
      }
     }
    }
   },
   "jevbench_score": 18.867176393547975,
   "key": "decision-2-0-vega-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "vllm-sr/Decision-2.0-Vega-27B@7aec49ae11a18741706da549ab626b9052795fe7",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9889705882352942,
    "decisive_rate": 0.7331536388140162,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.20136692745442,
     "cost": 29.632311028770005,
     "intelligence": 60.4093162408047,
     "speed": 83.80610555916236
    },
    "board_release": "v1.6.43",
    "capability": 71.30534158412956,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      18.452136544131196,
      19.07499323293124
     ],
     "B": [
      18.66694813760067,
      19.664978813028327
     ],
     "C": [
      15.74438986147704,
      19.07499323293124
     ]
    },
    "jevbench_score": 18.883756282485887,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 101,
    "rank": 136,
    "ranks": {
     "A": 136,
     "B": 132,
     "C": 118,
     "capability": 67
    },
    "scores": {
     "A": 18.883756282485887,
     "B": 19.308516568607676,
     "C": 18.883756282485887
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5303170211613178,
     "p50_s_raw": 0.1901585105806589,
     "p95_s_adjusted": 0.784971479140222,
     "p95_s_raw": 0.317485739570111
    },
    "views": {
     "Balanced 33:33:33": 16.931351780927532,
     "Emphasis on Accuracy 60:20:20": 18.419751896131185,
     "Emphasis on Cost 20:20:60": 13.537269231212232,
     "Emphasis on Speed 20:60:20": 20.397174151969452,
     "Intelligence only": 21.217536521841364
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 154,
   "ranked": true,
   "ranks": {
    "A": 154,
    "B": 150,
    "C": 135,
    "capability": 78
   },
   "repo": "https://huggingface.co/vllm-sr/Decision-2.0-Vega-27B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.867176393547975,
    "B": 19.294647028690232,
    "C": 18.867176393547975
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5422534752637148,
    "p50_s_raw": 0.1961267376318574,
    "p95_s_adjusted": 0.8528162744827569,
    "p95_s_raw": 0.3514081372413784
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.19756848350097,
      83.31136430016561
     ],
     "I": [
      55.369466346348005,
      64.07264251912545
     ],
     "capability": [
      67.8132497141865,
      72.6791469378467
     ],
     "composite": {
      "A": [
       18.436305597721148,
       19.058075982834907
      ],
      "B": [
       18.65398466828145,
       19.650592634580573
      ],
      "C": [
       15.730863308763542,
       19.058075982834907
      ]
     },
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.20136692745442,
     "capability": 71.30534158412956,
     "intelligence": 60.4093162408047
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8213333333333334,
      "calibration": 79.99303017339956,
      "cc": 77.1510851339958,
      "cc_by_type": {
       "choice": 77.1510851339958
      },
      "cc_tiered": 79.24346403575493,
      "cc_tiered_open": 83.11168946691484,
      "cc_tiered_sealed": 75.37523860459501,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 77.1510851339958
     },
     "noul": {
      "accuracy": 0.7173333333333334,
      "calibration": 84.09899360176757,
      "cc": 43.466666666666676,
      "cc_by_type": {
       "noul": 43.466666666666676
      },
      "cc_tiered": 44.39501780082543,
      "cc_tiered_open": 48.59696342305038,
      "cc_tiered_sealed": 40.19307217860048,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.466666666666676
     },
     "score": {
      "accuracy": 0.688,
      "calibration": 82.51207700719613,
      "cc": 56.084109797956,
      "cc_by_type": {
       "score": 56.084109797956
      },
      "cc_tiered": 57.58946688583374,
      "cc_tiered_open": 58.78284807646216,
      "cc_tiered_sealed": 56.396085695205315,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.084109797956
     }
    },
    "raw_on_A": {
     "calibration": 78.63363678590805,
     "capability": 71.50200013792696,
     "intelligence": 64.37036348994586
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.91358321758484,
    "Emphasis on Accuracy 60:20:20": 18.407129373776744,
    "Emphasis on Cost 20:20:60": 13.530450247979218,
    "Emphasis on Speed 20:60:20": 20.35081352069177,
    "Intelligence only": 21.217536521841364
   }
  },
  {
   "adapter_id": "images/decision-4b-v11/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 85.23934158755043,
    "cost": 63.07359395281681,
    "intelligence": 29.134168280172865,
    "speed": 93.91870342645993
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10557670546665254,
      "ece_score": 78.88465890666949,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.88465890666949,
      "score_before_time_limit": 78.88465890666949,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4339622641509434,
      "brier": 0.11841873108613687,
      "ece": 0.04782018815907789,
      "n": 371,
      "score": 90.43596236818442,
      "score_before_time_limit": 90.43596236818442,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9701897018970189,
      "interval90_width": 0.5756549232158987,
      "n": 369,
      "nrps": 0.11671345785768895,
      "score": 86.39740348779742,
      "score_before_time_limit": 86.39740348779742,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07766923619318136
     }
    },
    "score": 85.23934158755043
   },
   "capability": 57.18675493386165,
   "class": "unclassified",
   "composite_ci95": {
    "A": [
     12.443817844521899,
     27.252588677618228
    ],
    "B": [
     10.295326116443203,
     23.806683322108682
    ],
    "C": [
     8.641540169806873,
     18.92540880390155
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01701689039408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 4B v1.1 (FlyMyJev, Qwen3.5-4B + LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 30.696800084915584,
    "I_sealed": 27.571536475430143,
    "base": 29.134168280172865,
    "excess": 0.5518759670471964,
    "gap": 3.1252636094854402,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 60.19131316010312,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 56.59012268008808,
       "judge": 64.40342781806196,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 1.1380262249827418,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 13.043478260869556,
       "judge": -30.000000000000004,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 30.761060869660888,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.918373499249086,
       "hard": 31.938975913174172,
       "judge": 30.29541690533948,
       "standard": 17.025040414322312
      }
     },
     "sealed|choice": {
      "cc": 57.815173715536396,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 54.185520361990946,
       "judge": 56.238361266294234,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 2.4545445385650666,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -16.504854368932044,
       "judge": 10.526315789473696,
       "standard": 15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 22.444891172188967,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 55.42678299206407,
       "hard": 20.963579856299262,
       "judge": 15.25087787708388,
       "standard": 19.707587836688447
      }
     }
    }
   },
   "jevbench_score": 18.717253640537187,
   "key": "decision-4b-v11",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9428571428571428,
    "decisive_rate": 0.5660377358490566,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.23934158755043,
     "cost": 63.07359395281681,
     "intelligence": 29.134168280172865,
     "speed": 93.95641941218838
    },
    "board_release": "v1.6.43",
    "capability": 57.18675493386165,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 68,
    "capability_rank": 94,
    "composite_ci95": {
     "A": [
      12.444491289404162,
      27.25430281789905
     ],
     "B": [
      10.295694887525865,
      23.807726232797425
     ],
     "C": [
      8.642007839864,
      18.926599179096563
     ]
    },
    "jevbench_score": 18.718356274892123,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 104,
    "rank": 139,
    "ranks": {
     "A": 139,
     "B": 146,
     "C": 145,
     "capability": 148
    },
    "scores": {
     "A": 18.718356274892123,
     "B": 15.883627667705545,
     "C": 12.998858524230638
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.188053295109421,
     "p50_s_raw": 0.019026647554710507,
     "p95_s_adjusted": 0.21383417290635406,
     "p95_s_raw": 0.031917086453177035
    },
    "views": {
     "Balanced 33:33:33": 16.746653513023,
     "Emphasis on Accuracy 60:20:20": 13.11195414676558,
     "Emphasis on Cost 20:20:60": 18.346342760998827,
     "Emphasis on Speed 20:60:20": 20.675164334326784,
     "Intelligence only": 9.891630033656257
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 157,
   "ranked": true,
   "ranks": {
    "A": 157,
    "B": 164,
    "C": 163,
    "capability": 165
   },
   "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v11",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.717253640537187,
    "B": 15.882992492260271,
    "C": 12.9980928059286
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18865264891646802,
    "p50_s_raw": 0.019326324458234012,
    "p95_s_adjusted": 0.2150140054989606,
    "p95_s_raw": 0.032507002749480304
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.26750110051037,
      86.2633177195094
     ],
     "I": [
      24.784375611108008,
      34.07951912545236
     ],
     "capability": [
      52.99615856993229,
      59.129104174768116
     ],
     "composite": {
      "A": [
       12.443817844521899,
       27.252588677618228
      ],
      "B": [
       10.295326116443203,
       23.806683322108682
      ],
      "C": [
       8.641540169806873,
       18.92540880390155
      ]
     },
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.23934158755043,
     "capability": 57.18675493386165,
     "intelligence": 29.134168280172865
    },
    "per_type": {
     "choice": {
      "accuracy": 0.68,
      "calibration": 78.88465890666949,
      "cc": 59.07657038924623,
      "cc_by_type": {
       "choice": 59.07657038924623
      },
      "cc_tiered": 59.00324343781976,
      "cc_tiered_open": 60.19131316010312,
      "cc_tiered_sealed": 57.815173715536396,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.07657038924623
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 90.43596236818442,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 1.7962853817739042,
      "cc_tiered_open": 1.1380262249827418,
      "cc_tiered_sealed": 2.4545445385650666,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.47733333333333333,
      "calibration": 86.39740348779742,
      "cc": 26.584587548466242,
      "cc_by_type": {
       "score": 26.584587548466242
      },
      "cc_tiered": 26.602976020924928,
      "cc_tiered_open": 30.761060869660888,
      "cc_tiered_sealed": 22.444891172188967,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.584587548466242
     }
    },
    "raw_on_A": {
     "calibration": 85.6538943241602,
     "capability": 60.62425637183274,
     "intelligence": 35.59461841950529
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.745476758237032,
    "Emphasis on Accuracy 60:20:20": 13.111521301991274,
    "Emphasis on Cost 20:20:60": 18.345495356787538,
    "Emphasis on Speed 20:60:20": 20.67193611486748,
    "Intelligence only": 9.891630033656257
   }
  },
  {
   "adapter_id": "images/autojev-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 87.61544276592178,
    "cost": 29.365937020749882,
    "intelligence": 55.62422706919757,
    "speed": 88.30997360530876
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0794047531024999,
      "ece_score": 84.11904937950003,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 84.11904937950003,
      "score_before_time_limit": 84.11904937950003,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2964959568733153,
      "brier": 0.06886464230605442,
      "ece": 0.04566629583733653,
      "n": 371,
      "score": 90.8667408325327,
      "score_before_time_limit": 90.8667408325327,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 1.0,
      "interval90_width": 0.41666666666666674,
      "n": 369,
      "nrps": 0.0623820892760947,
      "score": 87.86053808573263,
      "score_before_time_limit": 87.86053808573263,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09020357450462646
     }
    },
    "score": 87.61544276592178
   },
   "capability": 71.61983491755967,
   "class": "unclassified",
   "composite_ci95": {
    "A": [
     17.94244546902315,
     18.705656075758146
    ],
    "B": [
     17.845116596293558,
     19.068651692394134
    ],
    "C": [
     12.74354434172492,
     18.574364028270207
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.22618758620689655
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "AutoJev-27B (denis-pplx, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 56.79842954079213,
    "I_sealed": 54.45002459760302,
    "base": 55.62422706919757,
    "excess": -0.2249826992491344,
    "gap": 2.3484049431891094,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.34292176773366,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 80.22412656558997,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 34.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 56.68383683393881,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.12454762185703,
       "hard": 56.31402454129169,
       "judge": 64.63259495424096,
       "standard": 41.2799688448207
      }
     },
     "sealed|choice": {
      "cc": 75.98476401702004,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 68.43891402714931,
       "judge": 78.58472998137803,
       "standard": 84.48084739499939
      }
     },
     "sealed|noul": {
      "cc": 33.77589728457613,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 12.621359223300965,
       "judge": 42.10526315789473,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 53.589412491212904,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.78624407613681,
       "hard": 41.90521556676637,
       "judge": 62.130233048417736,
       "standard": 56.54815971183675
      }
     }
    }
   },
   "jevbench_score": 18.45403610189411,
   "key": "autojev-27b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9885057471264368,
    "decisive_rate": 0.7035040431266847,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.61544276592178,
     "cost": 29.365937020749882,
     "intelligence": 55.62422706919757,
     "speed": 88.60757446254297
    },
    "board_release": "v1.6.43",
    "capability": 71.61983491755967,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      17.95132364974654,
      18.715305818590913
     ],
     "B": [
      17.85214158412037,
      19.07667324580802
     ],
     "C": [
      12.749857516255931,
      18.58395156303398
     ]
    },
    "jevbench_score": 18.463427917707932,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 105,
    "rank": 140,
    "ranks": {
     "A": 140,
     "B": 135,
     "C": 133,
     "capability": 65
    },
    "scores": {
     "A": 18.463427917707932,
     "B": 18.603781697586662,
     "C": 15.868570767928437
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3447543190326542,
     "p50_s_raw": 0.09737715951632708,
     "p95_s_adjusted": 0.3996987941907718,
     "p95_s_raw": 0.12484939709538592
    },
    "views": {
     "Balanced 33:33:33": 16.343749600153146,
     "Emphasis on Accuracy 60:20:20": 17.37362601712716,
     "Emphasis on Cost 20:20:60": 13.123431991427351,
     "Emphasis on Speed 20:60:20": 20.08099797467756,
     "Intelligence only": 19.187204603319074
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 158,
   "ranked": true,
   "ranks": {
    "A": 158,
    "B": 153,
    "C": 152,
    "capability": 75
   },
   "repo": "https://huggingface.co/denis-pplx/autojev-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.45403610189411,
    "B": 18.596152827941562,
    "C": 15.86049888146482
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3509636391885579,
    "p50_s_raw": 0.10048181959427893,
    "p95_s_adjusted": 0.42047532487194983,
    "p95_s_raw": 0.13523766243597493
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.93064258139867,
      88.05572734371755
     ],
     "I": [
      50.54754866954701,
      59.786942555692185
     ],
     "capability": [
      67.70288142885737,
      72.92248557654668
     ],
     "composite": {
      "A": [
       17.94244546902315,
       18.705656075758146
      ],
      "B": [
       17.845116596293558,
       19.068651692394134
      ],
      "C": [
       12.74354434172492,
       18.574364028270207
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.61544276592178,
     "capability": 71.61983491755967,
     "intelligence": 55.62422706919757
    },
    "per_type": {
     "choice": {
      "accuracy": 0.82,
      "calibration": 84.11904937950003,
      "cc": 76.98057084395099,
      "cc_by_type": {
       "choice": 76.98057084395099
      },
      "cc_tiered": 77.66384289237685,
      "cc_tiered_open": 79.34292176773366,
      "cc_tiered_sealed": 75.98476401702004,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.98057084395099
     },
     "noul": {
      "accuracy": 0.688,
      "calibration": 90.8667408325327,
      "cc": 37.59999999999999,
      "cc_by_type": {
       "noul": 37.59999999999999
      },
      "cc_tiered": 34.07221365264003,
      "cc_tiered_open": 34.36853002070393,
      "cc_tiered_sealed": 33.77589728457613,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 37.59999999999999
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 87.86053808573263,
      "cc": 53.92178222051296,
      "cc_by_type": {
       "score": 53.92178222051296
      },
      "cc_tiered": 55.136624662575855,
      "cc_tiered_open": 56.68383683393881,
      "cc_tiered_sealed": 53.589412491212904,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.92178222051296
     }
    },
    "raw_on_A": {
     "calibration": 85.84423247942522,
     "capability": 73.42644032760776,
     "intelligence": 61.00864817579031
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.333938285303557,
    "Emphasis on Accuracy 60:20:20": 17.366972512881098,
    "Emphasis on Cost 20:20:60": 13.119635302283465,
    "Emphasis on Speed 20:60:20": 20.054356936784252,
    "Intelligence only": 19.187204603319074
   }
  },
  {
   "adapter_id": "images/eikos-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 89.97714437568723,
    "cost": 28.689752882469676,
    "intelligence": 64.40270761245412,
    "speed": 87.88773525505371
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03950772533975903,
      "ece_score": 92.09845493204818,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.09845493204818,
      "score_before_time_limit": 92.09845493204818,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.19137466307277629,
      "brier": 0.05632135167655414,
      "ece": 0.06181891259769947,
      "n": 371,
      "score": 87.6362174804601,
      "score_before_time_limit": 87.6362174804601,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.3172990063233966,
      "n": 369,
      "nrps": 0.05868305372968562,
      "score": 90.19676071455342,
      "score_before_time_limit": 90.19676071455342,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06869086598962304
     }
    },
    "score": 89.97714437568723
   },
   "capability": 77.18992599407068,
   "class": "unclassified",
   "composite_ci95": {
    "A": [
     17.714785727449645,
     18.25498093096819
    ],
    "B": [
     18.082465357353502,
     18.988176754248187
    ],
    "C": [
     17.65042894286445,
     18.25498093096819
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.23823646551724137
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Eikos-27B (caiovicentino1, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 65.62981582129679,
    "I_sealed": 63.17559940361144,
    "base": 64.40270761245412,
    "excess": -0.1191712247528991,
    "gap": 2.4542164176853447,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 80.41823040534584,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 76.26895187870797,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 58.24775707384403,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 30.000000000000004,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 58.2234599847005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.643002416487,
       "hard": 61.32359173476558,
       "judge": 58.76732285181849,
       "standard": 49.49763096800006
      }
     },
     "sealed|choice": {
      "cc": 78.56119802347493,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 76.05498484960461,
       "hard": 74.5475113122172,
       "judge": 78.58472998137803,
       "standard": 87.80638009607095
      }
     },
     "sealed|noul": {
      "cc": 53.35348874031106,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 37.86407766990292,
       "judge": 57.89473684210527,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 57.61211144704836,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.33566285144035,
       "hard": 49.71122434442663,
       "judge": 64.06787863213144,
       "standard": 58.86845917247121
      }
     }
    }
   },
   "jevbench_score": 18.071526164361526,
   "key": "eikos-27b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.98,
    "decisive_rate": 0.8086253369272237,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.97714437568723,
     "cost": 28.689752882469676,
     "intelligence": 64.40270761245412,
     "speed": 88.08630815059152
    },
    "board_release": "v1.6.43",
    "capability": 77.18992599407068,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      17.720899810915583,
      18.261473654252256
     ],
     "B": [
      18.087561450267057,
      18.993796215955186
     ],
     "C": [
      17.656534367876468,
      18.261473654252256
     ]
    },
    "jevbench_score": 18.077889022432696,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 106,
    "rank": 141,
    "ranks": {
     "A": 141,
     "B": 134,
     "C": 123,
     "capability": 37
    },
    "scores": {
     "A": 18.077889022432696,
     "B": 18.627129011951496,
     "C": 18.077889022432696
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.37523653320968153,
     "p50_s_raw": 0.11261826660484076,
     "p95_s_adjusted": 0.4140607553534209,
     "p95_s_raw": 0.13203037767671047
    },
    "views": {
     "Balanced 33:33:33": 15.999272744012083,
     "Emphasis on Accuracy 60:20:20": 17.74116866920125,
     "Emphasis on Cost 20:20:60": 12.523731672975002,
     "Emphasis on Speed 20:60:20": 19.495440346386523,
     "Intelligence only": 21.20399692738122
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 160,
   "ranked": true,
   "ranks": {
    "A": 160,
    "B": 152,
    "C": 141,
    "capability": 44
   },
   "repo": "https://huggingface.co/caiovicentino1/eikos-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.071526164361526,
    "B": 18.621724387854368,
    "C": 18.071526164361526
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.38070584982633593,
    "p50_s_raw": 0.11535292491316795,
    "p95_s_adjusted": 0.4272055875975638,
    "p95_s_raw": 0.1386027937987819
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.83309061783127,
      90.19679304741126
     ],
     "I": [
      59.88680163370944,
      68.49583860207214
     ],
     "capability": [
      73.72374153392347,
      78.55486844196058
     ],
     "composite": {
      "A": [
       17.714785727449645,
       18.25498093096819
      ],
      "B": [
       18.082465357353502,
       18.988176754248187
      ],
      "C": [
       17.65042894286445,
       18.25498093096819
      ]
     },
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.97714437568723,
     "capability": 77.18992599407068,
     "intelligence": 64.40270761245412
    },
    "per_type": {
     "choice": {
      "accuracy": 0.836,
      "calibration": 92.09845493204818,
      "cc": 79.02674232448868,
      "cc_by_type": {
       "choice": 79.02674232448868
      },
      "cc_tiered": 79.48971421441038,
      "cc_tiered_open": 80.41823040534584,
      "cc_tiered_sealed": 78.56119802347493,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 79.02674232448868
     },
     "noul": {
      "accuracy": 0.784,
      "calibration": 87.6362174804601,
      "cc": 56.800000000000004,
      "cc_by_type": {
       "noul": 56.800000000000004
      },
      "cc_tiered": 55.80062290707754,
      "cc_tiered_open": 58.24775707384403,
      "cc_tiered_sealed": 53.35348874031106,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.800000000000004
     },
     "score": {
      "accuracy": 0.6853333333333333,
      "calibration": 90.19676071455342,
      "cc": 57.48579408299472,
      "cc_by_type": {
       "score": 57.48579408299472
      },
      "cc_tiered": 57.91778571587443,
      "cc_tiered_open": 58.2234599847005,
      "cc_tiered_sealed": 57.61211144704836,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.48579408299472
     }
    },
    "raw_on_A": {
     "calibration": 86.77546118929445,
     "capability": 77.43357682556581,
     "intelligence": 68.09169246183717
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.99262814913199,
    "Emphasis on Accuracy 60:20:20": 17.73626587017296,
    "Emphasis on Cost 20:20:60": 12.521288341177726,
    "Emphasis on Speed 20:60:20": 19.477690628960442,
    "Intelligence only": 21.20399692738122
   }
  },
  {
   "adapter_id": "images/ninfer-qwen3.8-27b/spec.json:ninfer_native.NInferNativeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Igor L. / NInfer contributors",
   "axes": {
    "calibration": 84.18557302696732,
    "cost": 29.118887844204664,
    "intelligence": 51.800126951055375,
    "speed": 90.16889017946136
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09587088648606537,
      "ece_score": 80.82582270278694,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.82582270278694,
      "score_before_time_limit": 80.82582270278694,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18328840970350402,
      "brier": 0.12070867053402283,
      "ece": 0.0947034093144267,
      "n": 371,
      "score": 81.05931813711467,
      "score_before_time_limit": 81.05931813711467,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.962059620596206,
      "interval90_width": 0.4135049683830172,
      "n": 369,
      "nrps": 0.07983756609019925,
      "score": 90.67157824100035,
      "score_before_time_limit": 90.67157824100035,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05336543454489691
     }
    },
    "score": 84.18557302696732
   },
   "capability": 67.99284998901135,
   "class": "native-logit",
   "composite_ci95": {
    "A": [
     14.650258599104475,
     18.0866888392295
    ],
    "B": [
     14.40864067769902,
     18.277595593631823
    ],
    "C": [
     10.173790693822555,
     15.96587798330431
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.2305174137931034
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "NInfer Qwen3.8-27B NVFP4",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 51.8273559598382,
    "I_sealed": 51.77289794227256,
    "base": 51.800126951055375,
    "excess": -2.5189296248726003,
    "gap": 0.0544580175656435,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 68.57305975575287,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 68.35860250494396,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 37.668046928916496,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 30.000000000000004,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 49.24096119484526,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.871723226731845,
       "hard": 32.99494797533653,
       "judge": 63.458675062497626,
       "standard": 54.0910358164409
      }
     },
     "sealed|choice": {
      "cc": 63.99439201930662,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 56.730769230769226,
       "judge": 63.68715083798884,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 42.068015862163264,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 20.388349514563096,
       "judge": 50.0,
       "standard": 60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 49.25628594534781,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.47448220301372,
       "hard": 39.79827642333791,
       "judge": 53.88993870966854,
       "standard": 51.612727714053584
      }
     }
    }
   },
   "jevbench_score": 17.70726239784648,
   "key": "ninfer-qwen3.8-27b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (engine and submitted 27B artifact/base)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8943894389438944,
    "decisive_rate": 0.816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.18557302696732,
     "cost": 29.118887844204664,
     "intelligence": 51.800126951055375,
     "speed": 90.47734239273063
    },
    "board_release": "v1.6.43",
    "capability": 67.99284998901135,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      14.657219833459003,
      18.09581018423065
     ],
     "B": [
      14.41402694236696,
      18.285046805435694
     ],
     "C": [
      10.178624884346531,
      15.973963875467245
     ]
    },
    "jevbench_score": 17.71600496541158,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 107,
    "rank": 143,
    "ranks": {
     "A": 143,
     "B": 138,
     "C": 141,
     "capability": 91
    },
    "scores": {
     "A": 17.71600496541158,
     "B": 17.686352887683018,
     "C": 13.20459060190698
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26002668901346626,
     "p50_s_raw": 0.05501334450673312,
     "p95_s_adjusted": 0.3445464937365614,
     "p95_s_raw": 0.09727324686828069
    },
    "views": {
     "Balanced 33:33:33": 15.726436779262448,
     "Emphasis on Accuracy 60:20:20": 16.41495956225673,
     "Emphasis on Cost 20:20:60": 12.713872701078419,
     "Emphasis on Speed 20:60:20": 19.536094431908698,
     "Intelligence only": 17.568730575960352
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 162,
   "ranked": true,
   "ranks": {
    "A": 162,
    "B": 156,
    "C": 159,
    "capability": 106
   },
   "repo": "https://github.com/igorls/ninfer",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.70726239784648,
    "B": 17.679381533909005,
    "C": 13.198074345802286
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26485669197281825,
    "p50_s_raw": 0.05742834598640911,
    "p95_s_adjusted": 0.36316169262281617,
    "p95_s_raw": 0.10658084631140809
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.72416061844064,
      85.97779338206968
     ],
     "I": [
      46.35923001218984,
      56.308300363548845
     ],
     "capability": [
      63.42455870096435,
      70.41651644718259
     ],
     "composite": {
      "A": [
       14.650258599104475,
       18.0866888392295
      ],
      "B": [
       14.40864067769902,
       18.277595593631823
      ],
      "C": [
       10.173790693822555,
       15.96587798330431
      ]
     },
     "seed": 72
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.18557302696732,
     "capability": 67.99284998901135,
     "intelligence": 51.800126951055375
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7266666666666667,
      "calibration": 80.82582270278694,
      "cc": 65.04457054081448,
      "cc_by_type": {
       "choice": 65.04457054081448
      },
      "cc_tiered": 66.28372588752974,
      "cc_tiered_open": 68.57305975575287,
      "cc_tiered_sealed": 63.99439201930662,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.04457054081448
     },
     "noul": {
      "accuracy": 0.7226666666666667,
      "calibration": 81.05931813711467,
      "cc": 44.53333333333334,
      "cc_by_type": {
       "noul": 44.53333333333334
      },
      "cc_tiered": 39.86803139553988,
      "cc_tiered_open": 37.668046928916496,
      "cc_tiered_sealed": 42.068015862163264,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 44.53333333333334
     },
     "score": {
      "accuracy": 0.6053333333333333,
      "calibration": 90.67157824100035,
      "cc": 49.00847977112264,
      "cc_by_type": {
       "score": 49.00847977112264
      },
      "cc_tiered": 49.24862357009654,
      "cc_tiered_open": 49.24096119484526,
      "cc_tiered_sealed": 49.25628594534781,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 49.00847977112264
     }
    },
    "raw_on_A": {
     "calibration": 79.59030598423057,
     "capability": 67.58012336905948,
     "intelligence": 55.569940753888396
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.717252026545937,
    "Emphasis on Accuracy 60:20:20": 16.40895429291751,
    "Emphasis on Cost 20:20:60": 12.710269865868778,
    "Emphasis on Speed 20:60:20": 19.51060023129281,
    "Intelligence only": 17.568730575960352
   }
  },
  {
   "adapter_id": "images/openjev-thinking/spec.json:openjev_thinking.ThinkingOpenJev",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "razorback16",
   "axes": {
    "calibration": 81.33711087001693,
    "cost": 28.51480924396573,
    "intelligence": 70.21463489688583,
    "speed": 72.91370874195911
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1238179439505489,
      "ece_score": 75.23641120989022,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 75.23641120989022,
      "score_before_time_limit": 75.23641120989022,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.04312668463611857,
      "brier": 0.06619006264403647,
      "ece": 0.07056572946447708,
      "n": 371,
      "score": 85.6578224961523,
      "score_before_time_limit": 85.88685410710458,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.8455284552845529,
      "interval90_width": 0.06526648599819332,
      "n": 369,
      "nrps": 0.05970047185852581,
      "score": 83.11709890400827,
      "score_before_time_limit": 83.11709890400827,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.13897877503065434
     }
    },
    "score": 81.33711087001693
   },
   "capability": 75.77587288345137,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     16.917160117114253,
     17.513665791884456
    ],
    "B": [
     17.623469766769954,
     18.542350172008582
    ],
    "C": [
     16.917160117114253,
     17.513665791884456
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.24145693965517237
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "OpenJev (thinking, BF16)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 68.15800897274833,
    "I_sealed": 72.2712608210233,
    "base": 70.21463489688583,
    "excess": -6.686639490713214,
    "gap": -4.11325184827497,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 71.32537528109535,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 67.9144385026738,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 68.51345755693583,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 80.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 64.63519408021384,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.65890690670581,
       "hard": 83.20743211500854,
       "judge": 51.76582051786083,
       "standard": 41.282921940907926
      }
     },
     "sealed|choice": {
      "cc": 72.81145995047186,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 66.4027149321267,
       "judge": 74.86033519553072,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 75.64956278293162,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 61.165048543689316,
       "judge": 81.57894736842107,
       "standard": 91.54929577464787
      }
     },
     "sealed|score": {
      "cc": 68.35275972966642,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 85.29967322252621,
       "hard": 58.44270568724672,
       "judge": 72.99731091086852,
       "standard": 72.73258429627272
      }
     }
    }
   },
   "jevbench_score": 17.27211236183207,
   "key": "openjev-thinking",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9322033898305084,
    "decisive_rate": 0.9567567567567568,
    "supported": true,
    "valid": 370
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.41345474033434,
     "cost": 28.51480924396573,
     "intelligence": 70.34621384425424,
     "speed": 72.55270827201397
    },
    "board_release": "v1.6.43",
    "capability": 75.87983429229429,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      16.909474945094924,
      17.502684272940023
     ],
     "B": [
      17.624044058024868,
      18.54119755452704
     ],
     "C": [
      16.909474945094924,
      17.502684272940023
     ]
    },
    "jevbench_score": 17.265218966202397,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 110,
    "rank": 146,
    "ranks": {
     "A": 146,
     "B": 136,
     "C": 126,
     "capability": 41
    },
    "scores": {
     "A": 17.265218966202397,
     "B": 18.156241294771178,
     "C": 17.265218966202397
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.8639664588030427,
     "p50_s_raw": 0.8569832294015214,
     "p95_s_adjusted": 2.980513406405225,
     "p95_s_raw": 1.4152567032026127
    },
    "views": {
     "Balanced 33:33:33": 15.470816250462342,
     "Emphasis on Accuracy 60:20:20": 17.77278752746646,
     "Emphasis on Cost 20:20:60": 12.207984243040777,
     "Emphasis on Speed 20:60:20": 17.94233922790431,
     "Intelligence only": 22.879243501891544
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 165,
   "ranked": true,
   "ranks": {
    "A": 165,
    "B": 154,
    "C": 144,
    "capability": 53
   },
   "repo": "https://github.com/razorback16/openjev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.27211236183207,
    "B": 18.15693743297266,
    "C": 17.27211236183207
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.9280300891026854,
    "p50_s_raw": 0.8890150445513427,
    "p95_s_adjusted": 2.651644620718434,
    "p95_s_raw": 1.250822310359217
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/diffusiongemma-26b-a4b-it, BF16; OpenJev think=512",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.19017694535097,
      83.80603684194338
     ],
     "I": [
      64.92265020833241,
      74.56158589207546
     ],
     "capability": [
      71.75831354064398,
      78.68433165919848
     ],
     "composite": {
      "A": [
       16.917160117114253,
       17.513665791884456
      ],
      "B": [
       17.623469766769954,
       18.542350172008582
      ],
      "C": [
       16.917160117114253,
       17.513665791884456
      ]
     },
     "seed": 81
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.33711087001693,
     "capability": 75.77587288345137,
     "intelligence": 70.21463489688583
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7946666666666666,
      "calibration": 75.23641120989022,
      "cc": 73.74079933309964,
      "cc_by_type": {
       "choice": 73.74079933309964
      },
      "cc_tiered": 72.0684176157836,
      "cc_tiered_open": 71.32537528109535,
      "cc_tiered_sealed": 72.81145995047186,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.74079933309964
     },
     "noul": {
      "accuracy": 0.88,
      "calibration": 85.6578224961523,
      "cc": 76.0,
      "cc_by_type": {
       "noul": 76.0
      },
      "cc_tiered": 72.08151016993372,
      "cc_tiered_open": 68.51345755693583,
      "cc_tiered_sealed": 75.64956278293162,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 76.0
     },
     "score": {
      "accuracy": 0.7786666666666666,
      "calibration": 83.11709890400827,
      "cc": 68.45852348469666,
      "cc_by_type": {
       "score": 68.45852348469666
      },
      "cc_tiered": 66.49397690494013,
      "cc_tiered_open": 64.63519408021384,
      "cc_tiered_sealed": 68.35275972966642,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.45852348469666
     }
    },
    "raw_on_A": {
     "calibration": 79.80009682824573,
     "capability": 76.44304334504619,
     "intelligence": 73.08598986184666
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.481028096596372,
    "Emphasis on Accuracy 60:20:20": 17.770519845316535,
    "Emphasis on Cost 20:20:60": 12.211798114590827,
    "Emphasis on Speed 20:60:20": 17.977662891938987,
    "Intelligence only": 22.836449062616822
   }
  },
  {
   "adapter_id": "images/open-jev-zefan-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zefan Cai (@Zefan_Cai)",
   "axes": {
    "calibration": 79.80507069734225,
    "cost": 33.05490478272432,
    "intelligence": 43.88591848753404,
    "speed": 74.31878072613569
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0965793794449889,
      "ece_score": 80.68412411100222,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.68412411100222,
      "score_before_time_limit": 80.68412411100222,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.1465943912444352,
      "ece": 0.11119868678055905,
      "n": 371,
      "score": 77.76026264388818,
      "score_before_time_limit": 77.76026264388818,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.926829268292683,
      "interval90_width": 0.46047877145438126,
      "n": 369,
      "nrps": 0.10981645578089093,
      "score": 80.97082533713633,
      "score_before_time_limit": 80.97082533713633,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.13538351873819116
     }
    },
    "score": 79.80507069734225
   },
   "capability": 61.84549459243814,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     12.009175545832976,
     21.472871382664703
    ],
    "B": [
     11.373882909186841,
     21.19627309905362
    ],
    "C": [
     8.339705240161791,
     14.911716237961603
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.17041317733990147
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open-Jev 9B (Zefan Cai)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 45.51928930503784,
    "I_sealed": 42.25254767003025,
    "base": 43.88591848753404,
    "excess": 0.6933539925693402,
    "gap": 3.266741635007584,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.95574336697866,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 64.1396665618119,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 35.9399585921325,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 38.66216595600236,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 45.958227241534885,
       "hard": 49.02462444279136,
       "judge": 29.24271270673138,
       "standard": 28.41839821356458
      }
     },
     "sealed|choice": {
      "cc": 56.97677837170926,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 48.076923076923066,
       "judge": 59.96275605214154,
       "standard": 66.7446729892844
      }
     },
     "sealed|noul": {
      "cc": 36.79892764867179,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48.0,
       "hard": 20.388349514563096,
       "judge": 44.736842105263165,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 32.98193698970971,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 51.23313360297058,
       "hard": 27.657644820832182,
       "judge": 32.23017056028181,
       "standard": 35.63257266497618
      }
     }
    }
   },
   "jevbench_score": 17.042643552961692,
   "key": "open-jev-zefan-9b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8637873754152824,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.80507069734225,
     "cost": 33.05490478272432,
     "intelligence": 43.88591848753404,
     "speed": 74.39486349899005
    },
    "board_release": "v1.6.43",
    "capability": 61.84549459243814,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      12.01117001715265,
      21.4767289933671
     ],
     "B": [
      11.375313778606227,
      21.19925799700656
     ],
     "C": [
      8.341090289689344,
      14.91439513428271
     ]
    },
    "jevbench_score": 17.045611746249723,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 111,
    "rank": 147,
    "ranks": {
     "A": 147,
     "B": 143,
     "C": 149,
     "capability": 121
    },
    "scores": {
     "A": 17.045611746249723,
     "B": 16.53766290698564,
     "C": 11.837230379340085
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.3850112770218401,
     "p50_s_raw": 0.6175056385109201,
     "p95_s_adjusted": 2.624583175848238,
     "p95_s_raw": 1.237291587924119
    },
    "views": {
     "Balanced 33:33:33": 15.193819551111247,
     "Emphasis on Accuracy 60:20:20": 15.024035626101695,
     "Emphasis on Cost 20:20:60": 13.257314615761418,
     "Emphasis on Speed 20:60:20": 18.03145694648765,
     "Intelligence only": 14.776357163022537
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 166,
   "ranked": true,
   "ranks": {
    "A": 166,
    "B": 161,
    "C": 169,
    "capability": 137
   },
   "repo": "https://github.com/Zefan-Cai/Open-Jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.042643552961692,
    "B": 16.535427677178838,
    "C": 11.835169134001175
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.4373612958937882,
    "p50_s_raw": 0.6436806479468942,
    "p95_s_adjusted": 2.5736884943442417,
    "p95_s_raw": 1.211844247172121
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-9B plus rank-8 LoRA and trained scalar decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      73.96736762912185,
      81.45578416794778
     ],
     "I": [
      37.82741036517579,
      48.601933428170696
     ],
     "capability": [
      56.87009089863343,
      64.27945845500723
     ],
     "composite": {
      "A": [
       12.009175545832976,
       21.472871382664703
      ],
      "B": [
       11.373882909186841,
       21.19627309905362
      ],
      "C": [
       8.339705240161791,
       14.911716237961603
      ]
     },
     "seed": 79
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.80507069734225,
     "capability": 61.84549459243814,
     "intelligence": 43.88591848753404
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6733333333333333,
      "calibration": 80.68412411100222,
      "cc": 58.22399893902218,
      "cc_by_type": {
       "choice": 58.22399893902218
      },
      "cc_tiered": 59.466260869343955,
      "cc_tiered_open": 61.95574336697866,
      "cc_tiered_sealed": 56.97677837170926,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.22399893902218
     },
     "noul": {
      "accuracy": 0.6933333333333334,
      "calibration": 77.76026264388818,
      "cc": 38.66666666666667,
      "cc_by_type": {
       "noul": 38.66666666666667
      },
      "cc_tiered": 36.369443120402146,
      "cc_tiered_open": 35.9399585921325,
      "cc_tiered_sealed": 36.79892764867179,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 38.66666666666667
     },
     "score": {
      "accuracy": 0.536,
      "calibration": 80.97082533713633,
      "cc": 35.27556488047482,
      "cc_by_type": {
       "score": 35.27556488047482
      },
      "cc_tiered": 35.82205147285603,
      "cc_tiered_open": 38.66216595600236,
      "cc_tiered_sealed": 32.98193698970971,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.27556488047482
     }
    },
    "raw_on_A": {
     "calibration": 76.09059647397694,
     "capability": 61.58634472209667,
     "intelligence": 47.0820929702164
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.190675240758345,
    "Emphasis on Accuracy 60:20:20": 15.02219081250179,
    "Emphasis on Cost 20:20:60": 13.255878145369945,
    "Emphasis on Speed 20:60:20": 18.02348758515275,
    "Intelligence only": 14.776357163022537
   }
  },
  {
   "adapter_id": "images/standardone-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking (myeongho12)",
   "axes": {
    "calibration": 84.01508092906352,
    "cost": 43.283250331904235,
    "intelligence": 32.80426604507879,
    "speed": 92.46998352781172
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1098197498466637,
      "ece_score": 78.03605003066725,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.03605003066725,
      "score_before_time_limit": 78.03605003066725,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.32345013477088946,
      "brier": 0.1399053250455738,
      "ece": 0.07067323668547182,
      "n": 371,
      "score": 85.86535266290564,
      "score_before_time_limit": 85.86535266290564,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.907859078590786,
      "interval90_width": 0.4114724480578141,
      "n": 369,
      "nrps": 0.10288890604189731,
      "score": 88.14384009361767,
      "score_before_time_limit": 88.14384009361767,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06711714604287465
     }
    },
    "score": 84.01508092906352
   },
   "capability": 58.40967348707115,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     11.013693486330085,
     24.234025771068143
    ],
    "B": [
     9.565594491333835,
     22.271817937890987
    ],
    "C": [
     7.648398254395891,
     16.829184563241764
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.07772456896551724
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 8B (Standard Thinking)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 30.33450693574831,
    "I_sealed": 35.27402515440927,
    "base": 32.80426604507879,
    "excess": -7.512905861099208,
    "gap": -4.9395182186609645,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 49.295789751985026,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 52.815350739226176,
       "judge": 44.62755438365195,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 8.598343685300202,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 13.043478260869556,
       "judge": -30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 33.109387369959705,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.66883781130674,
       "hard": 20.17576470368565,
       "judge": 52.27846833393313,
       "standard": 20.443286035874163
      }
     },
     "sealed|choice": {
      "cc": 51.12536859237399,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 39.42307692307691,
       "judge": 57.16945996275605,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 17.63077432402283,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": -12.621359223300976,
       "judge": 31.578947368421062,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 37.06593254683102,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 56.312646632383355,
       "hard": 30.71497074829812,
       "judge": 39.8913069210783,
       "standard": 35.90643753974974
      }
     }
    }
   },
   "jevbench_score": 16.909498058775554,
   "key": "standardone-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (jev-adapter server, LoRA and merged weights; base Ministral 3 8B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.900398406374502,
    "decisive_rate": 0.6765498652291105,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.01508092906352,
     "cost": 43.283250331904235,
     "intelligence": 32.80426604507879,
     "speed": 92.76442849252426
    },
    "board_release": "v1.6.43",
    "capability": 58.40967348707115,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      11.018270368438198,
      24.24546849303509
     ],
     "B": [
      9.568356102409696,
      22.27960999090091
     ],
     "C": [
      7.651576644748748,
      16.837130897941034
     ]
    },
    "jevbench_score": 16.917108299009165,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 112,
    "rank": 149,
    "ranks": {
     "A": 149,
     "B": 149,
     "C": 150,
     "capability": 140
    },
    "scores": {
     "A": 16.917108299009165,
     "B": 15.107984144533011,
     "C": 11.747991874311923
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21347399484366178,
     "p50_s_raw": 0.03173699742183089,
     "p95_s_adjusted": 0.24786326913163065,
     "p95_s_raw": 0.04893163456581533
    },
    "views": {
     "Balanced 33:33:33": 15.03402443438624,
     "Emphasis on Accuracy 60:20:20": 12.868191666930013,
     "Emphasis on Cost 20:20:60": 14.585944618765962,
     "Emphasis on Speed 20:60:20": 18.769748652230398,
     "Intelligence only": 10.581584191343778
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 168,
   "ranked": true,
   "ranks": {
    "A": 168,
    "B": 168,
    "C": 170,
    "capability": 158
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-8B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.909498058775554,
    "B": 15.103127851027393,
    "C": 11.742706985260801
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21601051054894924,
    "p50_s_raw": 0.03300525527447462,
    "p95_s_adjusted": 0.2621360577642917,
    "p95_s_raw": 0.056068028882145876
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.9630533878547,
      85.6304653046258
     ],
     "I": [
      27.551137017862857,
      38.18793043954382
     ],
     "capability": [
      53.64856195871698,
      60.83707776399651
     ],
     "composite": {
      "A": [
       11.013693486330085,
       24.234025771068143
      ],
      "B": [
       9.565594491333835,
       22.271817937890987
      ],
      "C": [
       7.648398254395891,
       16.829184563241764
      ]
     },
     "seed": 103
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.01508092906352,
     "capability": 58.40967348707115,
     "intelligence": 32.80426604507879
    },
    "per_type": {
     "choice": {
      "accuracy": 0.62,
      "calibration": 78.03605003066725,
      "cc": 51.40342733722988,
      "cc_by_type": {
       "choice": 51.40342733722988
      },
      "cc_tiered": 50.21057917217951,
      "cc_tiered_open": 49.295789751985026,
      "cc_tiered_sealed": 51.12536859237399,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.40342733722988
     },
     "noul": {
      "accuracy": 0.6026666666666667,
      "calibration": 85.86535266290564,
      "cc": 20.53333333333334,
      "cc_by_type": {
       "noul": 20.53333333333334
      },
      "cc_tiered": 13.114559004661515,
      "cc_tiered_open": 8.598343685300202,
      "cc_tiered_sealed": 17.63077432402283,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 20.53333333333334
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 88.14384009361767,
      "cc": 36.63215814216161,
      "cc_by_type": {
       "score": 36.63215814216161
      },
      "cc_tiered": 35.08765995839536,
      "cc_tiered_open": 33.109387369959705,
      "cc_tiered_sealed": 37.06593254683102,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.63215814216161
     }
    },
    "raw_on_A": {
     "calibration": 79.00702916038831,
     "capability": 58.75187777403198,
     "intelligence": 38.49672638767566
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.026011359123842,
    "Emphasis on Accuracy 60:20:20": 12.864668383151253,
    "Emphasis on Cost 20:20:60": 14.58141808431248,
    "Emphasis on Speed 20:60:20": 18.74728142290813,
    "Intelligence only": 10.581584191343778
   }
  },
  {
   "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/clef.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 86.72295444340945,
     "cost": 35.393298680703595,
     "intelligence": 67.9181689327805,
     "speed": 79.86099006822079
    },
    "label": "Workers AI price scenario (latency unmeasured)",
    "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.",
    "pricing_only": true,
    "usd_per_1000": 0.1424153201970443
   },
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Cloudflare",
   "axes": {
    "calibration": 91.57703482429629,
    "cost": 28.102157220114762,
    "intelligence": 59.08165501447566,
    "speed": 82.80534424231135
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04191843750000011,
      "ece_score": 91.61631249999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 91.61631249999998,
      "score_before_time_limit": 91.61631249999998,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.18400000000000005,
      "brier": 0.08179747616000003,
      "ece": 0.035773866666666626,
      "n": 375,
      "score": 92.84522666666668,
      "score_before_time_limit": 92.84522666666668,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.31244444444444447,
      "n": 375,
      "nrps": 0.07581989387555554,
      "score": 90.26956530622223,
      "score_before_time_limit": 90.26956530622223,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05939439999999997
     }
    },
    "score": 91.57703482429629
   },
   "capability": 75.32934491938597,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     16.26070958786801,
     16.994255552687996
    ],
    "B": [
     16.39142302319697,
     17.537622663163415
    ],
    "C": [
     13.000314490767794,
     16.994255552687996
    ]
   },
   "cost": {
    "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; self-hosted estimate, not measured hosted billing",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.24922681034482758
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Clef (Cloudflare, Qwen3.8-27B post-train with a joint schema head, multimodal, measured on text)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.04595985248707,
    "I_sealed": 61.117350176464235,
    "base": 59.08165501447566,
    "excess": -6.644777966415406,
    "gap": -4.0713903239771625,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 76.21786468827244,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 43.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 51.55148484848486,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.974848484848486,
       "hard": 50.55657407407409,
       "judge": 67.49000000000001,
       "standard": 24.421851851851862
      }
     },
     "sealed|choice": {
      "cc": 74.36979440941545,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 69.9660633484163,
       "judge": 74.86033519553072,
       "standard": 78.93829289321346
      }
     },
     "sealed|noul": {
      "cc": 52.76571930304361,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 32.038834951456316,
       "judge": 63.1578947368421,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 56.216536816933655,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 56.72936170212767,
       "hard": 49.606985294117656,
       "judge": 66.35520958083832,
       "standard": 53.971218274111656
      }
     }
    }
   },
   "jevbench_score": 16.73459266854105,
   "key": "clef",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Cloudflare/clef@2f3de3dd85f379784083b0814d997ab627200f0c",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9411764705882353,
    "decisive_rate": 0.816,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.57703482429629,
     "cost": 28.102157220114762,
     "intelligence": 59.08165501447566,
     "speed": 83.3998209667487
    },
    "board_release": "v1.6.43",
    "capability": 75.32934491938597,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      16.278742772255967,
      17.01395343115317
     ],
     "B": [
      16.406079295395777,
      17.554401382209043
     ],
     "C": [
      13.014738345348354,
      17.01395343115317
     ]
    },
    "jevbench_score": 16.753692861922005,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 113,
    "rank": 150,
    "ranks": {
     "A": 150,
     "B": 140,
     "C": 131,
     "capability": 42
    },
    "scores": {
     "A": 16.753692861922005,
     "B": 17.103724421252515,
     "C": 16.244762021167798
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5763746980577708,
     "p50_s_raw": 0.21318734902888536,
     "p95_s_adjusted": 0.7930726912803948,
     "p95_s_raw": 0.3215363456401974
    },
    "views": {
     "Balanced 33:33:33": 14.692533500484354,
     "Emphasis on Accuracy 60:20:20": 16.05926983891725,
     "Emphasis on Cost 20:20:60": 11.641983750272722,
     "Emphasis on Speed 20:60:20": 17.850768289881103,
     "Intelligence only": 18.663451480355018
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 169,
   "ranked": true,
   "ranks": {
    "A": 169,
    "B": 158,
    "C": 150,
    "capability": 54
   },
   "repo": "https://huggingface.co/Cloudflare/clef",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.73459266854105,
    "B": 17.08779577778536,
    "C": 16.22624203882187
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.610504524409771,
    "p50_s_raw": 0.23025226220488548,
    "p95_s_adjusted": 0.8585719365626574,
    "p95_s_raw": 0.3542859682813287
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.28138326498332,
      91.94843594390738
     ],
     "I": [
      53.626530822718266,
      63.68048125704343
     ],
     "capability": [
      70.47069839174009,
      77.24671214753253
     ],
     "composite": {
      "A": [
       16.26070958786801,
       16.994255552687996
      ],
      "B": [
       16.39142302319697,
       17.537622663163415
      ],
      "C": [
       13.000314490767794,
       16.994255552687996
      ]
     },
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.57703482429629,
     "capability": 75.32934491938597,
     "intelligence": 59.08165501447566
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 91.61631249999998,
      "cc": 75.61645652359255,
      "cc_by_type": {
       "choice": 75.61645652359255
      },
      "cc_tiered": 75.29382954884395,
      "cc_tiered_open": 76.21786468827244,
      "cc_tiered_sealed": 74.36979440941545,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.61645652359255
     },
     "noul": {
      "accuracy": 0.768,
      "calibration": 92.84522666666668,
      "cc": 53.6,
      "cc_by_type": {
       "noul": 53.6
      },
      "cc_tiered": 48.06712466187377,
      "cc_tiered_open": 43.36853002070393,
      "cc_tiered_sealed": 52.76571930304361,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.6
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 90.26956530622223,
      "cc": 54.4725614035088,
      "cc_by_type": {
       "score": 54.4725614035088
      },
      "cc_tiered": 53.88401083270926,
      "cc_tiered_open": 51.55148484848486,
      "cc_tiered_sealed": 56.216536816933655,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.4725614035088
     }
    },
    "raw_on_A": {
     "calibration": 88.38059707442306,
     "capability": 75.82665436300499,
     "intelligence": 63.27271165158693
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.67295114335051,
    "Emphasis on Accuracy 60:20:20": 16.045226392525798,
    "Emphasis on Cost 20:20:60": 11.6346016394147,
    "Emphasis on Speed 20:60:20": 17.798819978625144,
    "Intelligence only": 18.663451480355018
   }
  },
  {
   "adapter_id": "images/jev-qwen3.5-9b-base-nvfp4/spec.json:jev_choice_http.Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "WilfLin",
   "axes": {
    "calibration": 85.03288206087805,
    "cost": 47.592235403520306,
    "intelligence": 29.251131763478327,
    "speed": 93.41131234211913
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05599448584772476,
      "ece_score": 88.80110283045505,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.80110283045505,
      "score_before_time_limit": 88.80110283045505,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.47439353099730464,
      "brier": 0.15464738864390087,
      "ece": 0.10676843098382341,
      "n": 371,
      "score": 78.64631380323533,
      "score_before_time_limit": 78.64631380323533,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.975609756097561,
      "interval90_width": 0.6402439024390243,
      "n": 369,
      "nrps": 0.10255360461757664,
      "score": 87.65122954894377,
      "score_before_time_limit": 87.65122954894377,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07221090220177398
     }
    },
    "score": 85.03288206087805
   },
   "capability": 57.14200691217819,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     10.55100535128086,
     22.568544734606675
    ],
    "B": [
     8.922738700914902,
     20.044253440413932
    ],
    "C": [
     7.327087049500598,
     15.67260051014353
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05583743842364531
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JEV Qwen3.5-9B Base NVFP4",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 30.95041263176777,
    "I_sealed": 27.551850895188885,
    "base": 29.251131763478327,
    "excess": 0.8251740941406402,
    "gap": 3.398561736578884,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 55.43165802161466,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 47.153192827933324,
       "judge": 52.537903757415954,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 3.6694271911663243,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -21.739130434782606,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 33.750152682522334,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.19655618503351,
       "hard": 33.2215341463409,
       "judge": 26.48323152009081,
       "standard": 33.48456974727688
      }
     },
     "sealed|choice": {
      "cc": 47.891397457772825,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 66.74303451333975,
       "hard": 41.96832579185519,
       "judge": 45.065176908752335,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 24.0,
       "hard": -26.21359223300971,
       "judge": -18.42105263157895,
       "standard": -9.859154929577462
      }
     },
     "sealed|score": {
      "cc": 34.764155227793836,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 59.111326959327606,
       "hard": 22.60553224460797,
       "judge": 38.31832027118539,
       "standard": 41.57656776331136
      }
     }
    }
   },
   "jevbench_score": 15.970464603997094,
   "key": "jev-qwen3.5-9b-base-nvfp4",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 entrant and upstream checkpoint",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8923076923076924,
    "decisive_rate": 0.5256064690026954,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.03288206087805,
     "cost": 47.592235403520306,
     "intelligence": 29.251131763478327,
     "speed": 93.62779491136789
    },
    "board_release": "v1.6.43",
    "capability": 57.14200691217819,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 69,
    "capability_rank": 95,
    "composite_ci95": {
     "A": [
      10.554100896173676,
      22.576213316295057
     ],
     "B": [
      8.924509602976766,
      20.049092209565615
     ],
     "C": [
      7.329236733453941,
      15.67792591409379
     ]
    },
    "jevbench_score": 15.97555620862577,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 116,
    "rank": 154,
    "ranks": {
     "A": 154,
     "B": 156,
     "C": 154,
     "capability": 149
    },
    "scores": {
     "A": 15.97555620862577,
     "B": 13.864517363543307,
     "C": 11.094136255990119
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19534559999592602,
     "p50_s_raw": 0.02267279999796301,
     "p95_s_adjusted": 0.22203266686410641,
     "p95_s_raw": 0.03601633343205321
    },
    "views": {
     "Balanced 33:33:33": 14.120527794466737,
     "Emphasis on Accuracy 60:20:20": 11.548484256330752,
     "Emphasis on Cost 20:20:60": 14.368613309854894,
     "Emphasis on Speed 20:60:20": 17.771756183840008,
     "Intelligence only": 9.070269959573535
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 172,
   "ranked": true,
   "ranks": {
    "A": 172,
    "B": 173,
    "C": 173,
    "capability": 166
   },
   "repo": "https://huggingface.co/WIlfLin/JEV-Qwen3.5-9B-Base-NVFP4",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.970464603997094,
    "B": 13.861449157550995,
    "C": 11.090600419442426
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19610175802372395,
    "p50_s_raw": 0.02305087901186198,
    "p95_s_adjusted": 0.23248090025736018,
    "p95_s_raw": 0.04124045012868009
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-9B Base NVFP4 with compact BF16 decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.95850301005234,
      86.17596383374455
     ],
     "I": [
      24.784047457667274,
      33.682401804173935
     ],
     "capability": [
      52.832944781851424,
      58.83201835103813
     ],
     "composite": {
      "A": [
       10.55100535128086,
       22.568544734606675
      ],
      "B": [
       8.922738700914902,
       20.044253440413932
      ],
      "C": [
       7.327087049500598,
       15.67260051014353
      ]
     },
     "seed": 47
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.03288206087805,
     "capability": 57.14200691217819,
     "intelligence": 29.251131763478327
    },
    "per_type": {
     "choice": {
      "accuracy": 0.608,
      "calibration": 88.80110283045505,
      "cc": 49.86879872682661,
      "cc_by_type": {
       "choice": 49.86879872682661
      },
      "cc_tiered": 51.66152773969374,
      "cc_tiered_open": 55.43165802161466,
      "cc_tiered_sealed": 47.891397457772825,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.86879872682661
     },
     "noul": {
      "accuracy": 0.464,
      "calibration": 78.64631380323533,
      "cc": -7.199999999999996,
      "cc_by_type": {
       "noul": -7.199999999999996
      },
      "cc_tiered": 1.8347135955831622,
      "cc_tiered_open": 3.6694271911663243,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.5413333333333333,
      "calibration": 87.65122954894377,
      "cc": 35.19517866359504,
      "cc_by_type": {
       "score": 35.19517866359504
      },
      "cc_tiered": 34.25715395515809,
      "cc_tiered_open": 33.750152682522334,
      "cc_tiered_sealed": 34.764155227793836,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.19517866359504
     }
    },
    "raw_on_A": {
     "calibration": 80.94123658739136,
     "capability": 56.72695595498426,
     "intelligence": 32.51267532257717
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.115224343155328,
    "Emphasis on Accuracy 60:20:20": 11.546355426093731,
    "Emphasis on Cost 20:20:60": 14.365317962330064,
    "Emphasis on Speed 20:60:20": 17.75664203960864,
    "Intelligence only": 9.070269959573535
   }
  },
  {
   "adapter_id": "images/semif-qwen3.5-4b/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Theodore Lee (TheoLeeCJ)",
   "axes": {
    "calibration": 83.59960184071504,
    "cost": 63.07359395281681,
    "intelligence": 27.10113669038628,
    "speed": 90.43707331448911
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10233013188094879,
      "ece_score": 79.53397362381024,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 79.53397362381024,
      "score_before_time_limit": 79.53397362381024,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.38274932614555257,
      "brier": 0.1432523071034433,
      "ece": 0.08378278501118164,
      "n": 371,
      "score": 83.24344299776368,
      "score_before_time_limit": 83.24344299776368,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9701897018970189,
      "interval90_width": 0.6594399277326105,
      "n": 369,
      "nrps": 0.1212670700676031,
      "score": 88.0213889005712,
      "score_before_time_limit": 88.0213889005712,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.059152575960486446
     }
    },
    "score": 83.59960184071504
   },
   "capability": 55.35036926555066,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     9.240192823346929,
     23.170874693777435
    ],
    "B": [
     7.5205317882712315,
     19.98790589423405
    ],
    "C": [
     6.4168005717687,
     16.090885204012107
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01701689039408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 30.391025070826444,
    "I_sealed": 23.811248309946116,
    "base": 27.10113669038628,
    "excess": 4.006389118442085,
    "gap": 6.579776760880328,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 49.58254501782712,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 35.8288770053476,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 14.439613526570046,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 13.043478260869556,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 27.150916668082164,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.363221487255814,
       "hard": 12.79441887527647,
       "judge": 38.365166132705816,
       "standard": 21.436385647171228
      }
     },
     "sealed|choice": {
      "cc": 45.71562508055857,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 39.93212669683257,
       "judge": 45.99627560521416,
       "standard": 49.008498583569406
      }
     },
     "sealed|noul": {
      "cc": 2.17292780700555,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -22.330097087378643,
       "judge": 2.6315789473684292,
       "standard": 29.5774647887324
      }
     },
     "sealed|score": {
      "cc": 23.545192042274238,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.77035344302047,
       "hard": 13.542770599427534,
       "judge": 22.665102695300575,
       "standard": 24.757588248055008
      }
     }
    }
   },
   "jevbench_score": 15.508983430616452,
   "key": "semif-qwen3.5-4b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (code); Qwen3.5 weights Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9039301310043668,
    "decisive_rate": 0.6172506738544474,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.59960184071504,
     "cost": 63.07359395281681,
     "intelligence": 27.10113669038628,
     "speed": 90.76560119421038
    },
    "board_release": "v1.6.43",
    "capability": 55.35036926555066,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 81,
    "capability_rank": 107,
    "composite_ci95": {
     "A": [
      9.244572028428712,
      23.18413753262927
     ],
     "B": [
      7.522852111652043,
      19.995799931382237
     ],
     "C": [
      6.419841686408828,
      16.100095508770327
     ]
    },
    "jevbench_score": 15.517179498801362,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 121,
    "rank": 159,
    "ranks": {
     "A": 159,
     "B": 160,
     "C": 160,
     "capability": 162
    },
    "scores": {
     "A": 15.517179498801362,
     "B": 13.04205072010579,
     "C": 10.775819096389835
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25408456099685284,
     "p50_s_raw": 0.05204228049842641,
     "p95_s_adjusted": 0.32996023471932856,
     "p95_s_raw": 0.08998011735966428
    },
    "views": {
     "Balanced 33:33:33": 13.820858864848555,
     "Emphasis on Accuracy 60:20:20": 10.67791560389241,
     "Emphasis on Cost 20:20:60": 15.384875177789313,
     "Emphasis on Speed 20:60:20": 17.119455792629843,
     "Intelligence only": 7.962006198162448
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 176,
   "ranked": true,
   "ranks": {
    "A": 176,
    "B": 177,
    "C": 178,
    "capability": 181
   },
   "repo": "https://github.com/TheoLeeCJ/openjev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.508983430616452,
    "B": 13.037417990976522,
    "C": 10.770127382372536
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26757615598617124,
    "p50_s_raw": 0.05878807799308561,
    "p95_s_adjusted": 0.337944403081201,
    "p95_s_raw": 0.0939722015406005
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.25331982816681,
      85.80759674871754
     ],
     "I": [
      22.076409045997522,
      31.830294818443246
     ],
     "capability": [
      51.05521406162153,
      58.22417544189523
     ],
     "composite": {
      "A": [
       9.240192823346929,
       23.170874693777435
      ],
      "B": [
       7.5205317882712315,
       19.98790589423405
      ],
      "C": [
       6.4168005717687,
       16.090885204012107
      ]
     },
     "seed": 99
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.59960184071504,
     "capability": 55.35036926555066,
     "intelligence": 27.10113669038628
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5893333333333334,
      "calibration": 79.53397362381024,
      "cc": 47.48159866619932,
      "cc_by_type": {
       "choice": 47.48159866619932
      },
      "cc_tiered": 47.64908504919285,
      "cc_tiered_open": 49.58254501782712,
      "cc_tiered_sealed": 45.71562508055857,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 47.48159866619932
     },
     "noul": {
      "accuracy": 0.552,
      "calibration": 83.24344299776368,
      "cc": 10.40000000000001,
      "cc_by_type": {
       "noul": 10.40000000000001
      },
      "cc_tiered": 8.306270666787798,
      "cc_tiered_open": 14.439613526570046,
      "cc_tiered_sealed": 2.17292780700555,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 10.40000000000001
     },
     "score": {
      "accuracy": 0.49333333333333335,
      "calibration": 88.0213889005712,
      "cc": 25.84341933063762,
      "cc_by_type": {
       "score": 25.84341933063762
      },
      "cc_tiered": 25.3480543551782,
      "cc_tiered_open": 27.150916668082164,
      "cc_tiered_sealed": 23.545192042274238,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 25.84341933063762
     }
    },
    "raw_on_A": {
     "calibration": 83.04331750428547,
     "capability": 58.219039699698534,
     "intelligence": 33.3947618951116
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 13.812190328598344,
    "Emphasis on Accuracy 60:20:20": 10.674809999934535,
    "Emphasis on Cost 20:20:60": 15.378428953957561,
    "Emphasis on Speed 20:60:20": 17.095534022381575,
    "Intelligence only": 7.962006198162448
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Reza Sayar",
   "axes": {
    "calibration": 88.50232899798883,
    "cost": 28.231834531826323,
    "intelligence": 43.77513601921869,
    "speed": 72.92528840336786
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08046726298491785,
      "ece_score": 83.90654740301643,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 83.90654740301643,
      "score_before_time_limit": 83.90654740301643,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3045822102425876,
      "brier": 0.11954030846459725,
      "ece": 0.04318003176619355,
      "n": 371,
      "score": 91.36399364676129,
      "score_before_time_limit": 91.36399364676129,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.4652213188798555,
      "n": 369,
      "nrps": 0.08597620729262767,
      "score": 90.23644594418879,
      "score_before_time_limit": 90.23644594418879,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0546474369117983
     }
    },
    "score": 88.50232899798883
   },
   "capability": 66.13873250860377,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     8.549601979690372,
     14.585379958527726
    ],
    "B": [
     8.223454667943846,
     14.525738810108276
    ],
    "C": [
     5.937223597007205,
     10.128736082310919
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate. Frozen market reference qwen/qwen3.8-27b.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.2467585344827586
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bev / Bonsai 27B",
   "endpoint_condition": "evaluator-owned H100 GPU",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 46.71700087936099,
    "I_sealed": 40.833271159076396,
    "base": 43.77513601921869,
    "excess": 3.310342077846348,
    "gap": 5.883729720284592,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 63.31036599788361,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 47.153192827933324,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 30.23947550034507,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 46.6011611398543,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.472258307241056,
       "hard": 30.60094415356245,
       "judge": 62.977497036063276,
       "standard": 48.101542684431145
      }
     },
     "sealed|choice": {
      "cc": 56.58157537232299,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 66.74303451333975,
       "hard": 49.09502262443438,
       "judge": 53.44506517690876,
       "standard": 71.17871659071314
      }
     },
     "sealed|noul": {
      "cc": 20.312021130359053,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52.0,
       "hard": -0.9708737864077666,
       "judge": 26.315789473684205,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 45.60621697454716,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.9108636782132,
       "hard": 33.06846017805201,
       "judge": 52.300237792599205,
       "standard": 49.48837598862635
      }
     }
    }
   },
   "jevbench_score": 11.737862395830195,
   "key": "bev-bonsai-27b",
   "last_measured_on": "2026-10-04",
   "licence": "Not independently recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9147286821705426,
    "decisive_rate": 0.6954177897574124,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.50232899798883,
     "cost": 28.231834531826323,
     "intelligence": 43.77513601921869,
     "speed": 72.98222461463935
    },
    "board_release": "v1.6.43",
    "capability": 66.13873250860377,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      8.55065455714213,
      14.587291870035246
     ],
     "B": [
      8.22423019498041,
      14.527264406104972
     ],
     "C": [
      5.9379545535709255,
      10.130063798635586
     ]
    },
    "jevbench_score": 11.7393704396132,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 125,
    "rank": 164,
    "ranks": {
     "A": 164,
     "B": 163,
     "C": 165,
     "capability": 97
    },
    "scores": {
     "A": 11.7393704396132,
     "B": 11.515061943796129,
     "C": 8.152340583064722
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 2.1222116896882652,
     "p50_s_raw": 0.9861058448441327,
     "p95_s_adjusted": 2.371312748454511,
     "p95_s_raw": 1.1106563742272555
    },
    "views": {
     "Balanced 33:33:33": 10.186877852314403,
     "Emphasis on Accuracy 60:20:20": 10.385148364433647,
     "Emphasis on Cost 20:20:60": 8.555937486887114,
     "Emphasis on Speed 20:60:20": 12.296000304326956,
     "Intelligence only": 10.697460589059927
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 186,
   "ranked": true,
   "ranks": {
    "A": 186,
    "B": 182,
    "C": 187,
    "capability": 113
   },
   "repo": "https://huggingface.co/Reza2kn/Bev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 11.737862395830195,
    "B": 11.513901139808702,
    "C": 8.151293330437634
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.127405576966703,
    "p50_s_raw": 0.9887027884833515,
    "p95_s_adjusted": 2.3967396812047808,
    "p95_s_raw": 1.1233698406023904
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Reza2kn/Bev @ f01edf952e3f2da8147aa08ec6d6229c863d5d19. Unchanged Prism ML Ternary-Bonsai-2-27B PQ2_0 quantization of Qwen3.8-27B, with selected-token readout; not a fine-tune. GGUF SHA-256 3907dc1658db1f78a9826bf8d5bcb8dc65db0d466388937af57f2294fae62ec1. H100; CUDA backend compiled from the author-pinned source 9a9394a895b96003ca842a6041cb28ac49a108f7 with the author patch, rather than their prebuilt binary. The wire shim drops noul criteria because the submitted schema rejects them; that loses option descriptions for noul.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.75270117761126,
      88.77503324811279
     ],
     "I": [
      38.25187875678337,
      48.17301489659999
     ],
     "capability": [
      60.910547743797885,
      67.5609085693478
     ],
     "composite": {
      "A": [
       8.549601979690372,
       14.585379958527726
      ],
      "B": [
       8.223454667943846,
       14.525738810108276
      ],
      "C": [
       5.937223597007205,
       10.128736082310919
      ]
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.50232899798883,
     "capability": 66.13873250860377,
     "intelligence": 43.77513601921869
    },
    "per_type": {
     "choice": {
      "accuracy": 0.672,
      "calibration": 83.90654740301643,
      "cc": 58.053484648977374,
      "cc_by_type": {
       "choice": 58.053484648977374
      },
      "cc_tiered": 59.9459706851033,
      "cc_tiered_open": 63.31036599788361,
      "cc_tiered_sealed": 56.58157537232299,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.053484648977374
     },
     "noul": {
      "accuracy": 0.6293333333333333,
      "calibration": 91.36399364676129,
      "cc": 25.86666666666666,
      "cc_by_type": {
       "noul": 25.86666666666666
      },
      "cc_tiered": 25.27574831535206,
      "cc_tiered_open": 30.23947550034507,
      "cc_tiered_sealed": 20.312021130359053,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.86666666666666
     },
     "score": {
      "accuracy": 0.6,
      "calibration": 90.23644594418879,
      "cc": 45.43825770414722,
      "cc_by_type": {
       "score": 45.43825770414722
      },
      "cc_tiered": 46.103689057200725,
      "cc_tiered_open": 46.6011611398543,
      "cc_tiered_sealed": 45.60621697454716,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 45.43825770414722
     }
    },
    "raw_on_A": {
     "calibration": 80.9895865212274,
     "capability": 63.89601132732785,
     "intelligence": 46.8024361334283
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 10.185363815235588,
    "Emphasis on Accuracy 60:20:20": 10.384204181729109,
    "Emphasis on Cost 20:20:60": 8.555296612683984,
    "Emphasis on Speed 20:60:20": 12.29203041076874,
    "Intelligence only": 10.697460589059927
   }
  },
  {
   "adapter_id": "images/litjev/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zhengxu Yu",
   "axes": {
    "calibration": 89.9210925231847,
    "cost": 28.35809912639418,
    "intelligence": 43.28549422942267,
    "speed": 68.06914373137408
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.054566021834308726,
      "ece_score": 89.08679563313827,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 89.08679563313827,
      "score_before_time_limit": 89.08679563313827,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.33692722371967654,
      "brier": 0.09143307684062531,
      "ece": 0.03620776305464309,
      "n": 371,
      "score": 92.75844738907139,
      "score_before_time_limit": 92.75844738907139,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.981029810298103,
      "interval90_width": 0.534327009936766,
      "n": 369,
      "nrps": 0.08453630674044608,
      "score": 87.91803454734445,
      "score_before_time_limit": 87.91803454734445,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07855150115633251
     }
    },
    "score": 89.9210925231847
   },
   "capability": 66.60329337630368,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     8.744451216333523,
     14.762584129242377
    ],
    "B": [
     8.424587484217902,
     14.745790650644539
    ],
    "C": [
     6.072535566898279,
     10.2517945341961
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.2443787068965517
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "LitJev (Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 41.98760951977819,
    "I_sealed": 44.583378939067146,
    "base": 43.28549422942267,
    "excess": -5.1691570617272,
    "gap": -2.595769419288956,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 68.57906399743361,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 68.35860250494396,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 12.532781228433404,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 4.347826086956519,
       "judge": 0.0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 44.85098333346756,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.30288571369996,
       "hard": 36.97174671674506,
       "judge": 54.42696015026953,
       "standard": 39.519540151593404
      }
     },
     "sealed|choice": {
      "cc": 68.65215618863985,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 60.29411764705882,
       "judge": 66.4804469273743,
       "standard": 82.26382559428504
      }
     },
     "sealed|noul": {
      "cc": 24.95501594133015,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": -0.9708737864077666,
       "judge": 28.947368421052634,
       "standard": 49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 40.142964687231455,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.90525292252141,
       "hard": 32.24057461852555,
       "judge": 45.24652806264088,
       "standard": 35.91125564388415
      }
     }
    }
   },
   "jevbench_score": 11.45573083809696,
   "key": "litjev",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code); Apache-2.0 base weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.975609756097561,
    "decisive_rate": 0.6630727762803235,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.9210925231847,
     "cost": 28.35809912639418,
     "intelligence": 43.28549422942267,
     "speed": 68.17671225113584
    },
    "board_release": "v1.6.43",
    "capability": 66.60329337630368,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      8.746773766553591,
      14.766757637183968
     ],
     "B": [
      8.426311974704314,
      14.74912210872967
     ],
     "C": [
      6.074148448995549,
      10.25469280359998
     ]
    },
    "jevbench_score": 11.458886170296882,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 126,
    "rank": 165,
    "ranks": {
     "A": 165,
     "B": 164,
     "C": 166,
     "capability": 95
    },
    "scores": {
     "A": 11.458886170296882,
     "B": 11.238393678896292,
     "C": 7.957559840483947
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 3.0960148913785814,
     "p50_s_raw": 1.4730074456892908,
     "p95_s_adjusted": 4.9150250315666195,
     "p95_s_raw": 2.3825125157833096
    },
    "views": {
     "Balanced 33:33:33": 9.902804220753023,
     "Emphasis on Accuracy 60:20:20": 10.109112973826948,
     "Emphasis on Cost 20:20:60": 8.396442160471732,
     "Emphasis on Speed 20:60:20": 11.774987261719835,
     "Intelligence only": 10.435213933727047
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 187,
   "ranked": true,
   "ranks": {
    "A": 187,
    "B": 186,
    "C": 188,
    "capability": 110
   },
   "repo": "https://github.com/zhengxuyu/litjev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 11.45573083809696,
    "B": 11.235965478600717,
    "C": 7.955368637567337
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 3.1454328150488435,
    "p50_s_raw": 1.4977164075244218,
    "p95_s_adjusted": 4.959126807423309,
    "p95_s_raw": 2.4045634037116543
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B, frozen, read at the output head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.4774023239196,
      89.78804799226427
     ],
     "I": [
      38.555280103835365,
      48.498383980718195
     ],
     "capability": [
      62.48585734643129,
      68.14178334863401
     ],
     "composite": {
      "A": [
       8.744451216333523,
       14.762584129242377
      ],
      "B": [
       8.424587484217902,
       14.745790650644539
      ],
      "C": [
       6.072535566898279,
       10.2517945341961
      ]
     },
     "seed": 61
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.9210925231847,
     "capability": 66.60329337630368,
     "intelligence": 43.28549422942267
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7613333333333333,
      "calibration": 89.08679563313827,
      "cc": 69.47794208197946,
      "cc_by_type": {
       "choice": 69.47794208197946
      },
      "cc_tiered": 68.61561009303674,
      "cc_tiered_open": 68.57906399743361,
      "cc_tiered_sealed": 68.65215618863985,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.47794208197946
     },
     "noul": {
      "accuracy": 0.64,
      "calibration": 92.75844738907139,
      "cc": 28.000000000000004,
      "cc_by_type": {
       "noul": 28.000000000000004
      },
      "cc_tiered": 18.743898584881777,
      "cc_tiered_open": 12.532781228433404,
      "cc_tiered_sealed": 24.95501594133015,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 28.000000000000004
     },
     "score": {
      "accuracy": 0.5626666666666666,
      "calibration": 87.91803454734445,
      "cc": 41.34354803654359,
      "cc_by_type": {
       "score": 41.34354803654359
      },
      "cc_tiered": 42.496974010349504,
      "cc_tiered_open": 44.85098333346756,
      "cc_tiered_sealed": 40.142964687231455,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.34354803654359
     }
    },
    "raw_on_A": {
     "calibration": 89.32478284904047,
     "capability": 69.19315337175932,
     "intelligence": 49.06152389447817
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.899662285780803,
    "Emphasis on Accuracy 60:20:20": 10.107148204569665,
    "Emphasis on Cost 20:20:60": 8.39508668907996,
    "Emphasis on Speed 20:60:20": 11.766994127464926,
    "Intelligence only": 10.435213933727047
   }
  },
  {
   "adapter_id": "images/localjev-qwen3.5-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amith Chandrappa (amithgc)",
   "axes": {
    "calibration": 85.84286362718517,
    "cost": 59.269627658320445,
    "intelligence": 24.145363218466134,
    "speed": 85.6437855976805
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.039143749999999984,
      "ece_score": 92.17125,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 92.17125,
      "score_before_time_limit": 92.17125,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.648,
      "brier": 0.14289767218666674,
      "ece": 0.09039973333333337,
      "n": 375,
      "score": 81.92005333333331,
      "score_before_time_limit": 81.92005333333331,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.96,
      "interval90_width": 0.6557777777777777,
      "n": 375,
      "nrps": 0.13420838236888896,
      "score": 83.4372875482222,
      "score_before_time_limit": 83.4372875482222,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0985229333333334
     }
    },
    "score": 85.84286362718517
   },
   "capability": 54.99411342282565,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     6.9206676066563,
     15.229285149056274
    ],
    "B": [
     5.565582279578099,
     12.89581303236336
    ],
    "C": [
     4.806019171289097,
     10.57589246462241
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.022786576354679802
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "local-jev Qwen3.5-4B",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 26.448995848149686,
    "I_sealed": 21.841730588782582,
    "base": 24.145363218466134,
    "excess": 2.03387761692886,
    "gap": 4.607265259367104,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 52.23659737153346,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 41.49103491664046,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": -56.52173913043479,
       "judge": -30.000000000000004,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 27.1103901729156,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.470606060606066,
       "hard": 15.197777777777777,
       "judge": 40.368135593220345,
       "standard": 17.36888888888889
      }
     },
     "sealed|choice": {
      "cc": 46.955426808141326,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 42.477375565610856,
       "judge": 45.99627560521416,
       "standard": 50.1170094839266
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 12.00000000000001,
       "hard": -70.87378640776699,
       "judge": -18.42105263157895,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 18.56976495820642,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 56.15659574468085,
       "hard": 12.02813725490197,
       "judge": 13.700838323353281,
       "standard": 20.162994923857813
      }
     }
    }
   },
   "jevbench_score": 11.429477362870609,
   "key": "localjev-qwen3.5-4b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT code; Apache-2.0 Qwen weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9924242424242424,
    "decisive_rate": 0.352,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.84286362718517,
     "cost": 59.269627658320445,
     "intelligence": 24.145363218466134,
     "speed": 86.10068849774444
    },
    "board_release": "v1.6.43",
    "capability": 54.99411342282565,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 83,
    "capability_rank": 109,
    "composite_ci95": {
     "A": [
      6.925385954427524,
      15.24148889625826
     ],
     "B": [
      5.568022650808403,
      12.90281361284615
     ],
     "C": [
      4.8092958016857805,
      10.584367289068235
     ]
    },
    "jevbench_score": 11.438161297684031,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 127,
    "rank": 166,
    "ranks": {
     "A": 166,
     "B": 168,
     "C": 167,
     "capability": 164
    },
    "scores": {
     "A": 11.438161297684031,
     "B": 9.482181005106208,
     "C": 7.943167567836132
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.39534114915877583,
     "p50_s_raw": 0.1226705745793879,
     "p95_s_adjusted": 0.6208106089383364,
     "p95_s_raw": 0.2354053044691682
    },
    "views": {
     "Balanced 33:33:33": 10.008247462291514,
     "Emphasis on Accuracy 60:20:20": 7.6341780052667865,
     "Emphasis on Cost 20:20:60": 11.24977392613748,
     "Emphasis on Speed 20:60:20": 12.519995559254454,
     "Intelligence only": 5.630684842640914
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 188,
   "ranked": true,
   "ranks": {
    "A": 188,
    "B": 190,
    "C": 189,
    "capability": 183
   },
   "repo": "https://github.com/amithgc/local-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 11.429477362870609,
    "B": 9.477405474242415,
    "C": 7.937137057549032
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.42474671369418504,
    "p50_s_raw": 0.1373733568470925,
    "p95_s_adjusted": 0.6419355286285281,
    "p95_s_raw": 0.24596776431426406
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B text model, zero-shot next-token letter probabilities",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.50985112092962,
      86.15050973358518
     ],
     "I": [
      19.830618866047317,
      27.13596492581447
     ],
     "capability": [
      50.794515955411754,
      55.552509505023664
     ],
     "composite": {
      "A": [
       6.9206676066563,
       15.229285149056274
      ],
      "B": [
       5.565582279578099,
       12.89581303236336
      ],
      "C": [
       4.806019171289097,
       10.57589246462241
      ]
     },
     "seed": 62
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.84286362718517,
     "capability": 54.99411342282565,
     "intelligence": 24.145363218466134
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6026666666666667,
      "calibration": 92.17125,
      "cc": 49.186741566647385,
      "cc_by_type": {
       "choice": 49.186741566647385
      },
      "cc_tiered": 49.596012089837394,
      "cc_tiered_open": 52.23659737153346,
      "cc_tiered_sealed": 46.955426808141326,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.186741566647385
     },
     "noul": {
      "accuracy": 0.34933333333333333,
      "calibration": 81.92005333333331,
      "cc": -30.133333333333333,
      "cc_by_type": {
       "noul": -30.133333333333333
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.45066666666666666,
      "calibration": 83.4372875482222,
      "cc": 22.259017543859695,
      "cc_by_type": {
       "score": 22.259017543859695
      },
      "cc_tiered": 22.84007756556101,
      "cc_tiered_open": 27.1103901729156,
      "cc_tiered_sealed": 18.56976495820642,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.259017543859695
     }
    },
    "raw_on_A": {
     "calibration": 81.40355743693019,
     "capability": 54.83642941083503,
     "intelligence": 28.26930138473988
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.999383988105253,
    "Emphasis on Accuracy 60:20:20": 7.6310822090321055,
    "Emphasis on Cost 20:20:60": 11.243052646064552,
    "Emphasis on Speed 20:60:20": 12.495056067875103,
    "Intelligence only": 5.630684842640914
   }
  },
  {
   "adapter_id": "images/system-one-sg/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Sean Goedecke",
   "axes": {
    "calibration": 30.16473321941294,
    "cost": 44.96855803039237,
    "intelligence": 29.271508779158296,
    "speed": 90.51961611379383
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.5207165079484644,
      "ece_score": 0.0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0.0,
      "score_before_time_limit": 0.0,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.07547169811320753,
      "brier": 0.26938609640716954,
      "ece": 0.26647263490315864,
      "n": 371,
      "score": 46.70547301936827,
      "score_before_time_limit": 46.70547301936827,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.5718157181571816,
      "interval90_width": 0.08897922312556458,
      "n": 369,
      "nrps": 0.18780642533502573,
      "score": 43.78872663887056,
      "score_before_time_limit": 43.78872663887056,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.46820952094378154
     }
    },
    "score": 30.16473321941294
   },
   "capability": 29.718120999285617,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     5.877610065245049,
     15.677763264701111
    ],
    "B": [
     5.311781215377841,
     14.842956549913081
    ],
    "C": [
     4.081673656420172,
     10.887335600486882
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06829381896551724
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "system-one (Qwen3-8B, Sean Goedecke)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 32.67027386076885,
    "I_sealed": 25.87274369754774,
    "base": 29.271508779158296,
    "excess": 4.224142520782868,
    "gap": 6.797530163221111,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 45.21508118936821,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 30.16671909405474,
       "judge": 48.58272907053395,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 28.967563837129056,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 19.999999999999996,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 23.8281765558093,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.82115115650328,
       "hard": 4.701913776879685,
       "judge": 35.713306286008326,
       "standard": 23.25652021802299
      }
     },
     "sealed|choice": {
      "cc": 35.384978952910906,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 54.77052693814206,
       "hard": 19.57013574660632,
       "judge": 35.7541899441341,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 28.751778735776956,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": -2.9126213592232997,
       "judge": 44.736842105263165,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 13.48147340395537,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 53.14233360647993,
       "hard": 0.26200782926150135,
       "judge": 16.1186769737661,
       "standard": 16.134169097364737
      }
     }
    }
   },
   "jevbench_score": 11.022831386802604,
   "key": "system-one-sg",
   "last_measured_on": "2026-10-02",
   "licence": "no licence file in the repository as of 19 Sep; Qwen3 weights Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7288629737609329,
    "decisive_rate": 0.9245283018867925,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 30.16473321941294,
     "cost": 44.96855803039237,
     "intelligence": 29.271508779158296,
     "speed": 90.8867694864598
    },
    "board_release": "v1.6.43",
    "capability": 29.718120999285617,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      5.879902468266102,
      15.68532110290842
     ],
     "B": [
      5.313278874085685,
      14.848330476734906
     ],
     "C": [
      4.08326560296257,
      10.892584099241958
     ]
    },
    "jevbench_score": 11.027723483913586,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 128,
    "rank": 167,
    "ranks": {
     "A": 167,
     "B": 167,
     "C": 168,
     "capability": 230
    },
    "scores": {
     "A": 11.027723483913586,
     "B": 10.28901604793032,
     "C": 7.658141308273324
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24992560098180547,
     "p50_s_raw": 0.04996280049090274,
     "p95_s_adjusted": 0.3262212941306643,
     "p95_s_raw": 0.08811064706533216
    },
    "views": {
     "Balanced 33:33:33": 12.338652673779826,
     "Emphasis on Accuracy 60:20:20": 10.21232996468921,
     "Emphasis on Cost 20:20:60": 12.38939107483353,
     "Emphasis on Speed 20:60:20": 15.503079386401165,
     "Intelligence only": 8.114714247763947
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 189,
   "ranked": true,
   "ranks": {
    "A": 189,
    "B": 189,
    "C": 190,
    "capability": 252
   },
   "repo": "https://github.com/sgoedecke/system-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 11.022831386802604,
    "B": 10.285608751882629,
    "C": 7.654744018612918
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2562555870157667,
    "p50_s_raw": 0.053127793507883325,
    "p95_s_adjusted": 0.3462302790256217,
    "p95_s_raw": 0.09811513951281085
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3-8B (frozen, BF16), the model of the author's demos",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      26.715281574276638,
      33.82082873066771
     ],
     "I": [
      22.79600693038093,
      33.62231882810145
     ],
     "capability": [
      25.123779554066562,
      33.19293706149128
     ],
     "composite": {
      "A": [
       5.877610065245049,
       15.677763264701111
      ],
      "B": [
       5.311781215377841,
       14.842956549913081
      ],
      "C": [
       4.081673656420172,
       10.887335600486882
      ]
     },
     "seed": 105
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 30.16473321941294,
     "capability": 29.718120999285617,
     "intelligence": 29.271508779158296
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5133333333333333,
      "calibration": 0.0,
      "cc": 37.76228413364529,
      "cc_by_type": {
       "choice": 37.76228413364529
      },
      "cc_tiered": 40.30003007113956,
      "cc_tiered_open": 45.21508118936821,
      "cc_tiered_sealed": 35.384978952910906,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.76228413364529
     },
     "noul": {
      "accuracy": 0.6666666666666666,
      "calibration": 46.70547301936827,
      "cc": 33.33333333333333,
      "cc_by_type": {
       "noul": 33.33333333333333
      },
      "cc_tiered": 28.859671286453008,
      "cc_tiered_open": 28.967563837129056,
      "cc_tiered_sealed": 28.751778735776956,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.33333333333333
     },
     "score": {
      "accuracy": 0.45066666666666666,
      "calibration": 43.78872663887056,
      "cc": 16.56590736182537,
      "cc_by_type": {
       "score": 16.56590736182537
      },
      "cc_tiered": 18.654824979882335,
      "cc_tiered_open": 23.8281765558093,
      "cc_tiered_sealed": 13.48147340395537,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 16.56590736182537
     }
    },
    "raw_on_A": {
     "calibration": 29.76734864606182,
     "capability": 30.080860832336057,
     "intelligence": 30.394373018610295
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 12.330488677426247,
    "Emphasis on Accuracy 60:20:20": 10.208973261591055,
    "Emphasis on Cost 20:20:60": 12.384451008895631,
    "Emphasis on Speed 20:60:20": 15.479899388954921,
    "Intelligence only": 8.114714247763947
   }
  },
  {
   "adapter_id": "images/kev-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 70.7561729087212,
    "cost": 40.42054386073457,
    "intelligence": 29.26217941280341,
    "speed": 88.45850101024836
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14254951638954755,
      "ece_score": 71.49009672209048,
      "mean_tvd": null,
      "n": 736,
      "n_ece_hard": 306,
      "n_tvd": 0,
      "score": 71.49009672209048,
      "score_before_time_limit": 71.49009672209048,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.23450134770889486,
      "brier": 0.18970835579514836,
      "ece": 0.16199460916442043,
      "n": 371,
      "score": 67.60107816711592,
      "score_before_time_limit": 67.60107816711592,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8233695652173914,
      "interval90_width": 0.382925724637681,
      "n": 368,
      "nrps": 0.13575035733208884,
      "score": 73.17734383695722,
      "score_before_time_limit": 73.17734383695722,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.20035138296438337
     }
    },
    "score": 70.7561729087212
   },
   "capability": 50.0091761607623,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     6.284653394904355,
     16.47529143172706
    ],
    "B": [
     5.392786911175218,
     15.11290170704831
    ],
    "C": [
     4.364342635350246,
     11.44117460536601
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.09682374384236453
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 8B (research preview)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 28.412661059008222,
    "I_sealed": 30.111697766598596,
    "base": 29.26217941280341,
    "excess": -4.272424350028617,
    "gap": -1.6990367075903734,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 53.56342531883604,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 43.378420887071414,
       "judge": 48.58272907053395,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 15.995169082125601,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 13.043478260869556,
       "judge": 10.000000000000009,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 15.679388776063027,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 43.66574839302112,
       "hard": 33.21276294296097,
       "judge": 29.41280568734839,
       "standard": -53.98066473313996
      }
     },
     "sealed|choice": {
      "cc": 48.755934054679,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 41.96832579185519,
       "judge": 49.72067039106147,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 19.06361274442773,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -18.446601941747577,
       "judge": 50.0,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 22.51554650068906,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.4626547761159,
       "hard": 14.194388556502712,
       "judge": 23.600291166841224,
       "standard": 19.557191252120088
      }
     }
    }
   },
   "jevbench_score": 10.614631488944807,
   "key": "kev-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8063380281690141,
    "decisive_rate": 0.7654986522911051,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 70.7561729087212,
     "cost": 40.42054386073457,
     "intelligence": 29.26217941280341,
     "speed": 88.97143473610706
    },
    "board_release": "v1.6.43",
    "capability": 50.0091761607623,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      6.289030902475565,
      16.488978582767544
     ],
     "B": [
      5.395378045720429,
      15.122131514496889
     ],
     "C": [
      4.367382571163586,
      11.45067957136635
     ]
    },
    "jevbench_score": 10.62283914209564,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 130,
    "rank": 169,
    "ranks": {
     "A": 169,
     "B": 169,
     "C": 169,
     "capability": 173
    },
    "scores": {
     "A": 10.62283914209564,
     "B": 9.44790466894511,
     "C": 7.3769716264553065
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2989678788930178,
     "p50_s_raw": 0.07448393944650888,
     "p95_s_adjusted": 0.42386933271773153,
     "p95_s_raw": 0.13693466635886578
    },
    "views": {
     "Balanced 33:33:33": 9.572183421009896,
     "Emphasis on Accuracy 60:20:20": 8.080814281596565,
     "Emphasis on Cost 20:20:60": 9.355272676593998,
     "Emphasis on Speed 20:60:20": 12.08216537216834,
     "Intelligence only": 6.550045158904166
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 191,
   "ranked": true,
   "ranks": {
    "A": 191,
    "B": 191,
    "C": 191,
    "capability": 194
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 10.614631488944807,
    "B": 9.442709548873305,
    "C": 7.3712718673227835
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3028102615848184,
    "p50_s_raw": 0.07640513079240918,
    "p95_s_adjusted": 0.47095488999038937,
    "p95_s_raw": 0.16047744499519467
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-8B-Base + LoRA + learned pointer head; jaredpalmer/kev-8b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      65.36869404076147,
      74.24437950494625
     ],
     "I": [
      23.662522876918093,
      35.13692561855455
     ],
     "capability": [
      45.238684166857645,
      53.83400001933101
     ],
     "composite": {
      "A": [
       6.284653394904355,
       16.47529143172706
      ],
      "B": [
       5.392786911175218,
       15.11290170704831
      ],
      "C": [
       4.364342635350246,
       11.44117460536601
      ]
     },
     "seed": 55
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 70.7561729087212,
     "capability": 50.0091761607623,
     "intelligence": 29.26217941280341
    },
    "per_type": {
     "choice": {
      "accuracy": 0.612,
      "calibration": 71.49009672209048,
      "cc": 50.38034159696103,
      "cc_by_type": {
       "choice": 50.38034159696103
      },
      "cc_tiered": 51.15967968675752,
      "cc_tiered_open": 53.56342531883604,
      "cc_tiered_sealed": 48.755934054679,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.38034159696103
     },
     "noul": {
      "accuracy": 0.6106666666666667,
      "calibration": 67.60107816711592,
      "cc": 22.13333333333334,
      "cc_by_type": {
       "noul": 22.13333333333334
      },
      "cc_tiered": 17.529390913276664,
      "cc_tiered_open": 15.995169082125601,
      "cc_tiered_sealed": 19.06361274442773,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 22.13333333333334
     },
     "score": {
      "accuracy": 0.49866666666666665,
      "calibration": 73.17734383695722,
      "cc": 23.802092840863022,
      "cc_by_type": {
       "score": 23.802092840863022
      },
      "cc_tiered": 19.097467638376042,
      "cc_tiered_open": 15.679388776063027,
      "cc_tiered_sealed": 22.51554650068906,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.802092840863022
     }
    },
    "raw_on_A": {
     "calibration": 67.24332350856244,
     "capability": 50.76538650167667,
     "intelligence": 34.2874494947909
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.563298962777475,
    "Emphasis on Accuracy 60:20:20": 8.07701353082916,
    "Emphasis on Cost 20:20:60": 9.350178900784462,
    "Emphasis on Speed 20:60:20": 12.056717085026358,
    "Intelligence only": 6.550045158904166
   }
  },
  {
   "adapter_id": "images/decision-2b/spec.json:flymy_v15.FlyMyPackage",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "FlyMy.AI (@denti)",
   "axes": {
    "calibration": 86.77033341356835,
    "cost": 66.37275756963365,
    "intelligence": 22.875473256337628,
    "speed": 90.39467608500453
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.060081291070015405,
      "ece_score": 87.98374178599691,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.98374178599691,
      "score_before_time_limit": 87.98374178599691,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.6091644204851752,
      "brier": 0.15626137202130647,
      "ece": 0.06929060503851944,
      "n": 371,
      "score": 86.1418789922961,
      "score_before_time_limit": 86.1418789922961,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.646115627822945,
      "n": 369,
      "nrps": 0.13027713115375125,
      "score": 86.18537946241204,
      "score_before_time_limit": 86.18537946241204,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07300763979900397
     }
    },
    "score": 86.77033341356835
   },
   "capability": 54.82290333495299,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     6.070785321244929,
     14.301729550123063
    ],
    "B": [
     4.787007736285314,
     11.888763436425752
    ],
    "C": [
     4.215823139753423,
     9.931756632029904
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013210160098522166
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 2B (FlyMy.AI, v59)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 24.479797267586093,
    "I_sealed": 21.271149245089163,
    "base": 22.875473256337628,
    "excess": 0.6352603800586869,
    "gap": 3.2086480224969307,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 50.388518138122464,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 45.26580685750237,
       "judge": 40.672379696769944,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -56.52173913043479,
       "judge": -19.999999999999996,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 23.050873664635812,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 20.73971356294858,
       "hard": 15.558133148961872,
       "judge": 38.430985047283826,
       "standard": 16.12176767285528
      }
     },
     "sealed|choice": {
      "cc": 45.220796675917924,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 60.0916414160077,
       "hard": 37.38687782805428,
       "judge": 42.271880819366864,
       "standard": 57.876585786426915
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -4.0000000000000036,
       "hard": -53.398058252427184,
       "judge": -23.684210526315784,
       "standard": -18.309859154929576
      }
     },
     "sealed|score": {
      "cc": 18.59265105934957,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 27.03021479008173,
       "hard": 11.986008435765372,
       "judge": 25.699675889534046,
       "standard": 16.92661719587518
      }
     }
    }
   },
   "jevbench_score": 10.289709237720572,
   "key": "decision-2b",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9379310344827586,
    "decisive_rate": 0.3908355795148248,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.77033341356835,
     "cost": 66.37275756963365,
     "intelligence": 22.875473256337628,
     "speed": 90.39462801231525
    },
    "board_release": "v1.6.43",
    "capability": 54.82290333495299,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 84,
    "capability_rank": 110,
    "composite_ci95": {
     "A": [
      6.070784931981477,
      14.301728447276645
     ],
     "B": [
      4.787007542130122,
      11.888762823793376
     ],
     "C": [
      4.215822869431582,
      9.93175586616434
     ]
    },
    "jevbench_score": 10.289708493744515,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 131,
    "rank": 170,
    "ranks": {
     "A": 170,
     "B": 173,
     "C": 170,
     "capability": 165
    },
    "scores": {
     "A": 10.289708493744515,
     "B": 8.367001374285444,
     "C": 7.14563089843369
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.30028566285036506,
     "p50_s_raw": 0.07514283142518252,
     "p95_s_adjusted": 0.30409032410243525,
     "p95_s_raw": 0.07704516205121763
    },
    "views": {
     "Balanced 33:33:33": 8.990679348778704,
     "Emphasis on Accuracy 60:20:20": 6.6544725899582415,
     "Emphasis on Cost 20:20:60": 10.468172682915556,
     "Emphasis on Speed 20:60:20": 11.379622803195113,
     "Intelligence only": 4.788177641426015
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 192,
   "ranked": true,
   "ranks": {
    "A": 192,
    "B": 195,
    "C": 192,
    "capability": 184
   },
   "repo": "https://huggingface.co/flymy-ai/decision-2b-preview",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 10.289709237720572,
    "B": 8.367001767819637,
    "C": 7.14563141508373
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.30001436800230297,
    "p50_s_raw": 0.07500718400115147,
    "p95_s_adjusted": 0.3043619356816635,
    "p95_s_raw": 0.07718096784083173
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "openbmb/MiniCPM5-2B with a trained LoRA adapter and pointer head (26.2M trainable parameters)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.98496799808474,
      88.16729740402266
     ],
     "I": [
      18.649938213615208,
      26.096089404676093
     ],
     "capability": [
      50.712178970237844,
      56.11601608037113
     ],
     "composite": {
      "A": [
       6.070785321244929,
       14.301729550123063
      ],
      "B": [
       4.787007736285314,
       11.888763436425752
      ],
      "C": [
       4.215823139753423,
       9.931756632029904
      ]
     },
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.77033341356835,
     "capability": 54.82290333495299,
     "intelligence": 22.875473256337628
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5853333333333334,
      "calibration": 87.98374178599691,
      "cc": 46.9700557960649,
      "cc_by_type": {
       "choice": 46.9700557960649
      },
      "cc_tiered": 47.804657407020194,
      "cc_tiered_open": 50.388518138122464,
      "cc_tiered_sealed": 45.220796675917924,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 46.9700557960649
     },
     "noul": {
      "accuracy": 0.3626666666666667,
      "calibration": 86.1418789922961,
      "cc": -27.46666666666666,
      "cc_by_type": {
       "noul": -27.46666666666666
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.4266666666666667,
      "calibration": 86.18537946241204,
      "cc": 18.767847334171428,
      "cc_by_type": {
       "score": 18.767847334171428
      },
      "cc_tiered": 20.82176236199269,
      "cc_tiered_open": 23.050873664635812,
      "cc_tiered_sealed": 18.59265105934957,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 18.767847334171428
     }
    },
    "raw_on_A": {
     "calibration": 80.16146678308505,
     "capability": 51.30498083159781,
     "intelligence": 22.448494880110573
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 8.990680106093736,
    "Emphasis on Accuracy 60:20:20": 6.654472838884132,
    "Emphasis on Cost 20:20:60": 10.468173298921004,
    "Emphasis on Speed 20:60:20": 11.379624987030224,
    "Intelligence only": 4.788177641426015
   }
  },
  {
   "adapter_id": "images/opensourcejev-qwen35-4b-q4km/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "sabeel111",
   "axes": {
    "calibration": 73.92484912653177,
    "cost": 69.13657649738097,
    "intelligence": 23.217995107845883,
    "speed": 75.84507183307446
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20911848208469058,
      "ece_score": 58.17630358306188,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 58.17630358306188,
      "score_before_time_limit": 58.17630358306188,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.5121293800539084,
      "brier": 0.1552006734049543,
      "ece": 0.11625450134770884,
      "n": 371,
      "score": 76.74909973045823,
      "score_before_time_limit": 76.74909973045823,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9566395663956639,
      "interval90_width": 0.5600722673893404,
      "n": 369,
      "nrps": 0.11625319455925476,
      "score": 86.84914406607521,
      "score_before_time_limit": 86.84914406607521,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07338196205962055
     }
    },
    "score": 73.92484912653177
   },
   "capability": 48.571422117188824,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     6.468723462537756,
     14.540706945252797
    ],
    "B": [
     5.1878827057629895,
     12.31598399910567
    ],
    "C": [
     4.492169071206776,
     10.097713156425556
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0106851539408867
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "OpenSourceJev (Qwen3.5-4B Q4_K_M, native llama.cpp)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 24.81038701116274,
    "I_sealed": 21.62560320452903,
    "base": 23.217995107845883,
    "excess": 0.6113961641954666,
    "gap": 3.1847838066337104,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 41.32946193915506,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 24.50456118276188,
       "judge": 64.40342781806196,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -4.347826086956519,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 33.101699094333156,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.34023030303031,
       "hard": 17.52584074074074,
       "judge": 47.55760169491523,
       "standard": 25.450296296296294
      }
     },
     "sealed|choice": {
      "cc": 35.66176229147756,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 48.11913384081,
       "hard": 23.133484162895915,
       "judge": 42.271880819366864,
       "standard": 44.57445498214066
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 24.0,
       "hard": -30.097087378640776,
       "judge": -7.8947368421052655,
       "standard": -4.225352112676061
      }
     },
     "sealed|score": {
      "cc": 29.21504732210953,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.979268794326224,
       "hard": 18.989292892156882,
       "judge": 31.412349700598806,
       "standard": 31.488491878172564
      }
     }
    }
   },
   "jevbench_score": 10.238094968606852,
   "key": "opensourcejev-qwen35-4b-q4km",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (repository code); Apache-2.0 (Qwen/Qwen3.5-4B base and unsloth/Qwen3.5-4B-GGUF Q4_K_M conversion)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9502762430939227,
    "decisive_rate": 0.48787061994609165,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.92484912653177,
     "cost": 69.13657649738097,
     "intelligence": 23.217995107845883,
     "speed": 77.06362388237805
    },
    "board_release": "v1.6.43",
    "capability": 48.571422117188824,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 94,
    "capability_rank": 121,
    "composite_ci95": {
     "A": [
      6.483328619758367,
      14.579365175086792
     ],
     "B": [
      5.195391797769789,
      12.338152010991331
     ],
     "C": [
      4.502311541498866,
      10.124559149365826
     ]
    },
    "jevbench_score": 10.263493713193125,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 132,
    "rank": 171,
    "ranks": {
     "A": 171,
     "B": 171,
     "C": 171,
     "capability": 179
    },
    "scores": {
     "A": 10.263493713193125,
     "B": 8.482178412964995,
     "C": 7.127426189717446
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7395113246515393,
     "p50_s_raw": 0.29475566232576966,
     "p95_s_adjusted": 2.6588435446377843,
     "p95_s_raw": 1.2544217723188922
    },
    "views": {
     "Balanced 33:33:33": 9.174389111825883,
     "Emphasis on Accuracy 60:20:20": 6.882520870049428,
     "Emphasis on Cost 20:20:60": 10.842363781630336,
     "Emphasis on Speed 20:60:20": 11.176826098501683,
     "Intelligence only": 5.0064990418048145
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 193,
   "ranked": true,
   "ranks": {
    "A": 193,
    "B": 193,
    "C": 193,
    "capability": 200
   },
   "repo": "https://github.com/sabeel111/OpenSourceJev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 10.238094968606852,
    "B": 8.468288775003014,
    "C": 7.109788172643645
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7913347106426954,
    "p50_s_raw": 0.3206673553213477,
    "p95_s_adjusted": 3.2895205278880892,
    "p95_s_raw": 1.5697602639440447
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.41904601995815,
      77.70943106496745
     ],
     "I": [
      19.379013533491335,
      26.692735209928347
     ],
     "capability": [
      44.976218992193104,
      51.58836189201757
     ],
     "composite": {
      "A": [
       6.468723462537756,
       14.540706945252797
      ],
      "B": [
       5.1878827057629895,
       12.31598399910567
      ],
      "C": [
       4.492169071206776,
       10.097713156425556
      ]
     },
     "seed": 84
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.92484912653177,
     "capability": 48.571422117188824,
     "intelligence": 23.217995107845883
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5093333333333333,
      "calibration": 58.17630358306188,
      "cc": 37.25074126351086,
      "cc_by_type": {
       "choice": 37.25074126351086
      },
      "cc_tiered": 38.495612115316305,
      "cc_tiered_open": 41.32946193915506,
      "cc_tiered_sealed": 35.66176229147756,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.25074126351086
     },
     "noul": {
      "accuracy": 0.45866666666666667,
      "calibration": 76.74909973045823,
      "cc": -8.266666666666666,
      "cc_by_type": {
       "noul": -8.266666666666666
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.544,
      "calibration": 86.84914406607521,
      "cc": 30.80778824561413,
      "cc_by_type": {
       "score": 30.80778824561413
      },
      "cc_tiered": 31.15837320822134,
      "cc_tiered_open": 33.101699094333156,
      "cc_tiered_sealed": 29.21504732210953,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.80778824561413
     }
    },
    "raw_on_A": {
     "calibration": 73.9791977813484,
     "capability": 49.500913389632025,
     "intelligence": 25.022628997915646
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.147342758228765,
    "Emphasis on Accuracy 60:20:20": 6.873373308665869,
    "Emphasis on Cost 20:20:60": 10.81967943725586,
    "Emphasis on Speed 20:60:20": 11.104824965676269,
    "Intelligence only": 5.0064990418048145
   }
  },
  {
   "adapter_id": "images/open-alternative-jev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "IkerMoel",
   "axes": {
    "calibration": 75.94384178347806,
    "cost": 63.27859776193043,
    "intelligence": 22.34610151162883,
    "speed": 90.86432704430521
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.15207997234343346,
      "ece_score": 69.58400553131331,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 69.58400553131331,
      "score_before_time_limit": 69.58400553131331,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4,
      "brier": 0.1703546052656481,
      "ece": 0.0957881739676287,
      "n": 375,
      "score": 80.84236520647426,
      "score_before_time_limit": 80.84236520647426,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8933333333333333,
      "interval90_width": 0.5735555555555554,
      "n": 375,
      "nrps": 0.14203249622577255,
      "score": 77.40515461264664,
      "score_before_time_limit": 77.40515461264664,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1549322057606473
     }
    },
    "score": 75.94384178347806
   },
   "capability": 49.14497164755345,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     6.319511462595341,
     15.808691476316723
    ],
    "B": [
     5.047963685923234,
     13.390389942400779
    ],
    "C": [
     4.388549626802321,
     10.97825796966439
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0167512315270936
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "open-alternative-jev (Qwen3.5-4B, IkerMoel)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 25.032294599047653,
    "I_sealed": 19.659908424210006,
    "base": 22.34610151162883,
    "excess": 2.7989985323994033,
    "gap": 5.372386174837647,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 51.43479500294909,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 64.97297297297297,
       "hard": 43.378420887071414,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.8184955141476893,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": 4.347826086956519,
       "judge": -19.999999999999996,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 22.843593280046196,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 40.9867728497478,
       "hard": 12.570359188934388,
       "judge": 28.72169648259133,
       "standard": 25.50131687360131
      }
     },
     "sealed|choice": {
      "cc": 44.97077692151599,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 57.43108417707487,
       "hard": 39.93212669683257,
       "judge": 40.409683426443216,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 0.6994422333695622,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 32.00000000000001,
       "hard": -14.563106796116509,
       "judge": 2.6315789473684292,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 13.309506117744464,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 44.90364228457862,
       "hard": 10.653374925310422,
       "judge": 6.439637126414932,
       "standard": 13.129503906189765
      }
     }
    }
   },
   "jevbench_score": 9.429815011124056,
   "key": "open-alternative-jev",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code and weights)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8666666666666667,
    "decisive_rate": 0.6,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 75.94384178347806,
     "cost": 63.27859776193043,
     "intelligence": 22.34610151162883,
     "speed": 91.30216983336405
    },
    "board_release": "v1.6.43",
    "capability": 49.14497164755345,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 93,
    "capability_rank": 120,
    "composite_ci95": {
     "A": [
      6.323116186163818,
      15.819609663846963
     ],
     "B": [
      5.049803290865915,
      13.396655191684498
     ],
     "C": [
      4.391052907058209,
      10.985840044338168
     ]
    },
    "jevbench_score": 9.435692571873393,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 134,
    "rank": 173,
    "ranks": {
     "A": 173,
     "B": 175,
     "C": 174,
     "capability": 178
    },
    "scores": {
     "A": 9.435692571873393,
     "B": 7.716442780206986,
     "C": 6.5525642860231885
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2366679080063477,
     "p50_s_raw": 0.04333395400317386,
     "p95_s_adjusted": 0.3130715797946323,
     "p95_s_raw": 0.08153578989731614
    },
    "views": {
     "Balanced 33:33:33": 8.37993057455771,
     "Emphasis on Accuracy 60:20:20": 6.202799597209423,
     "Emphasis on Cost 20:20:60": 9.685497012220603,
     "Emphasis on Speed 20:60:20": 10.691355012814217,
     "Intelligence only": 4.463394698403469
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 195,
   "ranked": true,
   "ranks": {
    "A": 195,
    "B": 196,
    "C": 196,
    "capability": 198
   },
   "repo": "https://github.com/ikermoel/open-alternative-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 9.429815011124056,
    "B": 7.713297441806441,
    "C": 6.548482646613928
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25050918796332555,
    "p50_s_raw": 0.050254593981662765,
    "p95_s_adjusted": 0.3271475230052602,
    "p95_s_raw": 0.08857376150263008
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.67388183281524,
      78.71286748398254
     ],
     "I": [
      19.129525848760927,
      27.488222507229114
     ],
     "capability": [
      45.43710032043718,
      52.3768690725618
     ],
     "composite": {
      "A": [
       6.319511462595341,
       15.808691476316723
      ],
      "B": [
       5.047963685923234,
       13.390389942400779
      ],
      "C": [
       4.388549626802321,
       10.97825796966439
      ]
     },
     "seed": 74
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.94384178347806,
     "capability": 49.14497164755345,
     "intelligence": 22.34610151162883
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5866666666666667,
      "calibration": 69.58400553131331,
      "cc": 47.14057008610969,
      "cc_by_type": {
       "choice": 47.14057008610969
      },
      "cc_tiered": 48.20278596223254,
      "cc_tiered_open": 51.43479500294909,
      "cc_tiered_sealed": 44.97077692151599,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 47.14057008610969
     },
     "noul": {
      "accuracy": 0.52,
      "calibration": 80.84236520647426,
      "cc": 4.0000000000000036,
      "cc_by_type": {
       "noul": 4.0000000000000036
      },
      "cc_tiered": 0.7589688737586258,
      "cc_tiered_open": 0.8184955141476893,
      "cc_tiered_sealed": 0.6994422333695622,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 4.0000000000000036
     },
     "score": {
      "accuracy": 0.4106666666666667,
      "calibration": 77.40515461264664,
      "cc": 17.046666794552323,
      "cc_by_type": {
       "score": 17.046666794552323
      },
      "cc_tiered": 18.07654969889533,
      "cc_tiered_open": 22.843593280046196,
      "cc_tiered_sealed": 13.309506117744464,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 17.046666794552323
     }
    },
    "raw_on_A": {
     "calibration": 76.45819741974654,
     "capability": 51.99334325152506,
     "intelligence": 27.528489083303576
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 8.373750135533221,
    "Emphasis on Accuracy 60:20:20": 6.200767037449664,
    "Emphasis on Cost 20:20:60": 9.680542148093952,
    "Emphasis on Speed 20:60:20": 10.673264060965625,
    "Intelligence only": 4.463394698403469
   }
  },
  {
   "adapter_id": "images/raw-qwen3-8b/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 33.721548452619835,
    "cost": 45.53515592313027,
    "intelligence": 26.54912920590792,
    "speed": 88.245839907826
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.43645983930453747,
      "ece_score": 12.708032139092506,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 12.708032139092506,
      "score_before_time_limit": 12.708032139092506,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.07816711590296499,
      "brier": 0.2740805017981965,
      "ece": 0.2700267098593338,
      "n": 371,
      "score": 45.99465802813324,
      "score_before_time_limit": 45.99465802813324,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.5474254742547425,
      "interval90_width": 0.14972899728997294,
      "n": 369,
      "nrps": 0.21398318753780107,
      "score": 42.461955190633766,
      "score_before_time_limit": 42.461955190633766,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.46838885432476185
     }
    },
    "score": 33.721548452619835
   },
   "capability": 30.13533882926388,
   "class": "raw-logit-control",
   "composite_ci95": {
    "A": [
     5.524357631906432,
     13.598066110776006
    ],
    "B": [
     4.900288808693698,
     12.593335762848557
    ],
    "C": [
     3.8363594666016896,
     9.443101465816667
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0653875104679803
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 8B direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 31.420181182575643,
    "I_sealed": 21.6780772292402,
    "base": 26.54912920590792,
    "excess": 7.168716310897201,
    "gap": 9.742103953335445,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 51.791553839411165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 33.941491034916645,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 34.967563837129056,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 39.99999999999999,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 7.50142587118671,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 42.05293172396295,
       "hard": 3.1607638000191285,
       "judge": 1.3576889332232334,
       "standard": 8.122602494078969
      }
     },
     "sealed|choice": {
      "cc": 41.65523094061268,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 57.43108417707487,
       "hard": 31.787330316742068,
       "judge": 39.47858472998139,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 22.61439253816204,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": -6.796116504854366,
       "judge": 31.578947368421062,
       "standard": 49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 0.7646082089458659,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.377225661492794,
       "hard": -19.926430375972124,
       "judge": 5.951813095928216,
       "standard": 9.559569322034855
      }
     }
    }
   },
   "jevbench_score": 9.296450430510369,
   "key": "raw-qwen3-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7105263157894737,
    "decisive_rate": 0.921832884097035,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 33.721548452619835,
     "cost": 45.53515592313027,
     "intelligence": 26.54912920590792,
     "speed": 88.65282990878164
    },
    "board_release": "v1.6.43",
    "capability": 30.13533882926388,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      5.526896473528715,
      13.605697544750722
     ],
     "B": [
      4.901937683715855,
      12.59857128804635
     ],
     "C": [
      3.838122551061608,
      9.448401072743554
     ]
    },
    "jevbench_score": 9.30125972789427,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 135,
    "rank": 174,
    "ranks": {
     "A": 174,
     "B": 172,
     "C": 175,
     "capability": 229
    },
    "scores": {
     "A": 9.30125972789427,
     "B": 8.458419096926727,
     "C": 6.4592081443710185
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.30506142796948554,
     "p50_s_raw": 0.07753071398474276,
     "p95_s_adjusted": 0.4470228368416429,
     "p95_s_raw": 0.14851141842082144
    },
    "views": {
     "Balanced 33:33:33": 9.893413746086136,
     "Emphasis on Accuracy 60:20:20": 7.995042950818957,
     "Emphasis on Cost 20:20:60": 10.181976971451938,
     "Emphasis on Speed 20:60:20": 12.509103386849494,
     "Intelligence only": 6.208182595906101
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 196,
   "ranked": true,
   "ranks": {
    "A": 196,
    "B": 194,
    "C": 197,
    "capability": 251
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-8B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 9.296450430510369,
    "B": 8.455236895474457,
    "C": 6.455868354521089
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.31334649920463564,
    "p50_s_raw": 0.08167324960231781,
    "p95_s_adjusted": 0.47795957052148874,
    "p95_s_raw": 0.16397978526074436
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-8B BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      28.041560709983848,
      38.19473887695128
     ],
     "I": [
      21.71577559015635,
      30.903657696346237
     ],
     "capability": [
      25.49211567892071,
      33.98867030681561
     ],
     "composite": {
      "A": [
       5.524357631906432,
       13.598066110776006
      ],
      "B": [
       4.900288808693698,
       12.593335762848557
      ],
      "C": [
       3.8363594666016896,
       9.443101465816667
      ]
     },
     "seed": 97
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 33.721548452619835,
     "capability": 30.13533882926388,
     "intelligence": 26.54912920590792
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5613333333333334,
      "calibration": 12.708032139092506,
      "cc": 43.90079857525836,
      "cc_by_type": {
       "choice": 43.90079857525836
      },
      "cc_tiered": 46.723392390011924,
      "cc_tiered_open": 51.791553839411165,
      "cc_tiered_sealed": 41.65523094061268,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 43.90079857525836
     },
     "noul": {
      "accuracy": 0.648,
      "calibration": 45.99465802813324,
      "cc": 29.600000000000005,
      "cc_by_type": {
       "noul": 29.600000000000005
      },
      "cc_tiered": 28.79097818764555,
      "cc_tiered_open": 34.967563837129056,
      "cc_tiered_sealed": 22.61439253816204,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 29.600000000000005
     },
     "score": {
      "accuracy": 0.3973333333333333,
      "calibration": 42.461955190633766,
      "cc": 3.4025581754630996,
      "cc_by_type": {
       "score": 3.4025581754630996
      },
      "cc_tiered": 4.133017040066288,
      "cc_tiered_open": 7.50142587118671,
      "cc_tiered_sealed": 0.7646082089458659,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 3.4025581754630996
     }
    },
    "raw_on_A": {
     "calibration": 38.60246474366818,
     "capability": 35.16687873306722,
     "intelligence": 31.73129272246627
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.886160452338725,
    "Emphasis on Accuracy 60:20:20": 7.9921998004878185,
    "Emphasis on Cost 20:20:60": 10.177366129893914,
    "Emphasis on Speed 20:60:20": 12.488250713721934,
    "Intelligence only": 6.208182595906101
   }
  },
  {
   "adapter_id": "images/decider-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 68.52438190259257,
    "cost": 64.91766177740826,
    "intelligence": 20.654576369710338,
    "speed": 94.47902882434015
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.18752718750000008,
      "ece_score": 62.49456249999999,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 62.49456249999999,
      "score_before_time_limit": 62.49456249999999,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.31999999999999995,
      "brier": 0.21768861922666663,
      "ece": 0.1706301333333334,
      "n": 375,
      "score": 65.87397333333331,
      "score_before_time_limit": 65.87397333333331,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8746666666666667,
      "interval90_width": 0.5113333333333334,
      "n": 375,
      "nrps": 0.14404540251111114,
      "score": 77.20460987444443,
      "score_before_time_limit": 77.20460987444443,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1559312000000001
     }
    },
    "score": 68.52438190259257
   },
   "capability": 44.58947913615145,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     4.4701547319886465,
     13.008818378906568
    ],
    "B": [
     3.511831293736488,
     10.889929972365865
    ],
    "C": [
     3.10427411943656,
     9.033901652018448
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014771046798029557
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-2b (Mapika)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 22.803908585466385,
    "I_sealed": 18.50524415395429,
    "base": 20.654576369710338,
    "excess": 1.7252767890738503,
    "gap": 4.298664431512094,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 37.01745599452806,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 24.50456118276188,
       "judge": 28.80685563612393,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 4.469289164941341,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 4.347826086956519,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 26.92498059692975,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.77454545454546,
       "hard": 25.165648148148158,
       "judge": 23.80372881355931,
       "standard": 18.20074074074074
      }
     },
     "sealed|choice": {
      "cc": 39.65284081929382,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 58.7613627965413,
       "hard": 29.24208144796379,
       "judge": 35.7541899441341,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 0.12111092718806482,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -28.155339805825243,
       "judge": 10.526315789473696,
       "standard": 21.126760563380277
      }
     },
     "sealed|score": {
      "cc": 15.741780715380987,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.834113475177304,
       "hard": 4.744583333333363,
       "judge": 10.707844311377212,
       "standard": 26.740913705583736
      }
     }
    }
   },
   "jevbench_score": 7.669657300415898,
   "key": "decider-2b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7764705882352941,
    "decisive_rate": 0.68,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 68.52438190259257,
     "cost": 64.91766177740826,
     "intelligence": 20.654576369710338,
     "speed": 94.55945724666857
    },
    "board_release": "v1.6.43",
    "capability": 44.58947913615145,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 101,
    "capability_rank": 129,
    "composite_ci95": {
     "A": [
      4.470553979976864,
      13.010288397584645
     ],
     "B": [
      3.512028417919115,
      10.890754224642697
     ],
     "C": [
      3.1045513749839326,
      9.034922498322668
     ]
    },
    "jevbench_score": 7.6704332123718,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 139,
    "rank": 178,
    "ranks": {
     "A": 178,
     "B": 179,
     "C": 180,
     "capability": 188
    },
    "scores": {
     "A": 7.6704332123718,
     "B": 6.209608793546842,
     "C": 5.326689730813751
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.17949020988307893,
     "p50_s_raw": 0.014745104941539466,
     "p95_s_adjusted": 0.1949905256042257,
     "p95_s_raw": 0.022495262802112847
    },
    "views": {
     "Balanced 33:33:33": 6.881302219979439,
     "Emphasis on Accuracy 60:20:20": 4.98303010129531,
     "Emphasis on Cost 20:20:60": 8.110241448995462,
     "Emphasis on Speed 20:60:20": 8.93000700376026,
     "Intelligence only": 3.5245921294252027
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 200,
   "ranked": true,
   "ranks": {
    "A": 200,
    "B": 201,
    "C": 202,
    "capability": 210
   },
   "repo": "https://huggingface.co/Mapika/decider-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 7.669657300415898,
    "B": 6.209201969508958,
    "C": 5.326150903066597
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18164706099778413,
    "p50_s_raw": 0.01582353049889207,
    "p95_s_adjusted": 0.19627669751644133,
    "p95_s_raw": 0.02313834875822067
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-2B-Base with a trained decision readout, 1.9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.32738945440369,
      71.2629933305287
     ],
     "I": [
      16.778817874747006,
      25.441954749747556
     ],
     "capability": [
      40.31419112262012,
      47.6366012348925
     ],
     "composite": {
      "A": [
       4.4701547319886465,
       13.008818378906568
      ],
      "B": [
       3.511831293736488,
       10.889929972365865
      ],
      "C": [
       3.10427411943656,
       9.033901652018448
      ]
     },
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 68.52438190259257,
     "capability": 44.58947913615145,
     "intelligence": 20.654576369710338
    },
    "per_type": {
     "choice": {
      "accuracy": 0.532,
      "calibration": 62.49456249999999,
      "cc": 40.149484194272596,
      "cc_by_type": {
       "choice": 40.149484194272596
      },
      "cc_tiered": 38.335148406910946,
      "cc_tiered_open": 37.01745599452806,
      "cc_tiered_sealed": 39.65284081929382,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 40.149484194272596
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 65.87397333333331,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 2.295200046064703,
      "cc_tiered_open": 4.469289164941341,
      "cc_tiered_sealed": 0.12111092718806482,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.41333333333333333,
      "calibration": 77.20460987444443,
      "cc": 20.228824561403492,
      "cc_by_type": {
       "score": 20.228824561403492
      },
      "cc_tiered": 21.33338065615537,
      "cc_tiered_open": 26.92498059692975,
      "cc_tiered_sealed": 15.741780715380987,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 20.228824561403492
     }
    },
    "raw_on_A": {
     "calibration": 62.755705376823364,
     "capability": 43.8983266486265,
     "intelligence": 25.040947920429637
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.880469605342288,
    "Emphasis on Accuracy 60:20:20": 4.982768119709398,
    "Emphasis on Cost 20:20:60": 8.109547484927889,
    "Emphasis on Speed 20:60:20": 8.927483472504635,
    "Intelligence only": 3.5245921294252027
   }
  },
  {
   "adapter_id": "a3/candidates/nemotron-diffusion-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pst2154",
   "axes": {
    "calibration": 74.53740737677316,
    "cost": 55.49170925809556,
    "intelligence": 20.38848344571935,
    "speed": 93.55093866077343
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10832159242229476,
      "ece_score": 78.33568151554104,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.33568151554104,
      "score_before_time_limit": 78.33568151554104,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3881401617250674,
      "brier": 0.22241599402070583,
      "ece": 0.17236678386479326,
      "n": 371,
      "score": 65.52664322704135,
      "score_before_time_limit": 65.52664322704135,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9214092140921409,
      "interval90_width": 0.6684733514001803,
      "n": 369,
      "nrps": 0.14545408533817059,
      "score": 79.74989738773714,
      "score_before_time_limit": 79.74989738773714,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.12977398345354327
     }
    },
    "score": 74.53740737677316
   },
   "capability": 47.46294541124625,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     4.166408924156154,
     10.200690713939366
    ],
    "B": [
     3.2795753619277317,
     8.48009811794044
    ],
    "C": [
     2.8933395306639964,
     7.083812995791227
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03045157019704434
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nemotron Diffusion 8B (pst2154, optimized vLLM)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 22.973528132452522,
    "I_sealed": 17.803438758986175,
    "base": 20.38848344571935,
    "excess": 2.596701731028103,
    "gap": 5.170089373466347,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 45.141807097909854,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 37.72972972972973,
       "hard": 32.05410506448569,
       "judge": 48.58272907053395,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -21.739130434782606,
       "judge": -40.0,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 23.77877729944772,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.848253383055216,
       "hard": 4.46243778457357,
       "judge": 33.03604774332046,
       "standard": 37.99081262158317
      }
     },
     "sealed|choice": {
      "cc": 40.98559356961426,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 25.169683257918543,
       "judge": 50.65176908752329,
       "standard": 47.89998768321223
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -43.689320388349515,
       "judge": 5.263157894736836,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 12.424722707344273,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 42.099894800900906,
       "hard": 9.652238984201755,
       "judge": 2.7258103611165985,
       "standard": 17.6804726261925
      }
     }
    }
   },
   "jevbench_score": 7.294934989446692,
   "key": "nemotron-diffusion-8b",
   "last_measured_on": "2026-10-04",
   "licence": "MIT (application code); modified vLLM runtime Apache-2.0; nvidia/Nemotron-Labs-Diffusion-8B keeps its upstream licence",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7665198237885462,
    "decisive_rate": 0.6118598382749326,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 74.53740737677316,
     "cost": 55.49170925809556,
     "intelligence": 20.38848344571935,
     "speed": 93.8305724848098
    },
    "board_release": "v1.6.43",
    "capability": 47.46294541124625,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 97,
    "capability_rank": 124,
    "composite_ci95": {
     "A": [
      4.1676910325965215,
      10.20449077801364
     ],
     "B": [
      3.2802107600784733,
      8.482218509265811
     ],
     "C": [
      2.894229883747585,
      7.086451929176138
     ]
    },
    "jevbench_score": 7.29748476516348,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 140,
    "rank": 179,
    "ranks": {
     "A": 179,
     "B": 181,
     "C": 181,
     "capability": 183
    },
    "scores": {
     "A": 7.29748476516348,
     "B": 5.930430684235979,
     "C": 5.06769775358575
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1939013559371233,
     "p50_s_raw": 0.02195067796856165,
     "p95_s_adjusted": 0.21348231588490307,
     "p95_s_raw": 0.03174115794245154
    },
    "views": {
     "Balanced 33:33:33": 6.417819358792639,
     "Emphasis on Accuracy 60:20:20": 4.7285860819837495,
     "Emphasis on Cost 20:20:60": 7.307751777390142,
     "Emphasis on Speed 20:60:20": 8.394346515316924,
     "Intelligence only": 3.3901175711211273
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 201,
   "ranked": true,
   "ranks": {
    "A": 201,
    "B": 203,
    "C": 203,
    "capability": 205
   },
   "repo": "https://github.com/pst2154/Nemotron_Jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 7.294934989446692,
    "B": 5.929083363060404,
    "C": 5.065927076004647
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19840535297989845,
    "p50_s_raw": 0.024202676489949226,
    "p95_s_adjusted": 0.22251164717599747,
    "p95_s_raw": 0.036255823587998746
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "nvidia/Nemotron-Labs-Diffusion-8B @ 16c67f05 (unchanged weights) served by github.com/pst2154/Nemotron_Jev @ cdc36d60 on the author's digest-pinned modified vLLM runtime: BF16, candidate-only output projection, direct candidate logits, CUDA graphs, one masked forward pass per decision, nothing generated. Disclosed by the author: the inherited option-ordering rule was selected using public benchmark examples in earlier 14B experiments. Our earlier v1.4.3 run of this system used a recipe that left the base image's older 14B application in place; this v1.5 row is a fresh measurement of the 8B.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.2662042332654,
      77.36530331396067
     ],
     "I": [
      16.421783930074305,
      23.311693043206645
     ],
     "capability": [
      43.39466614794243,
      49.39379257235283
     ],
     "composite": {
      "A": [
       4.166408924156154,
       10.200690713939366
      ],
      "B": [
       3.2795753619277317,
       8.48009811794044
      ],
      "C": [
       2.8933395306639964,
       7.083812995791227
      ]
     },
     "seed": 70
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 74.53740737677316,
     "capability": 47.46294541124625,
     "intelligence": 20.38848344571935
    },
    "per_type": {
     "choice": {
      "accuracy": 0.54,
      "calibration": 78.33568151554104,
      "cc": 41.17256993454144,
      "cc_by_type": {
       "choice": 41.17256993454144
      },
      "cc_tiered": 43.06370033376206,
      "cc_tiered_open": 45.141807097909854,
      "cc_tiered_sealed": 40.98559356961426,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 41.17256993454144
     },
     "noul": {
      "accuracy": 0.464,
      "calibration": 65.52664322704135,
      "cc": -7.199999999999996,
      "cc_by_type": {
       "noul": -7.199999999999996
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.4266666666666667,
      "calibration": 79.74989738773714,
      "cc": 16.44673378084317,
      "cc_by_type": {
       "score": 16.44673378084317
      },
      "cc_tiered": 18.101750003396,
      "cc_tiered_open": 23.77877729944772,
      "cc_tiered_sealed": 12.424722707344273,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 16.44673378084317
     }
    },
    "raw_on_A": {
     "calibration": 75.92799396845535,
     "capability": 48.766079474284126,
     "intelligence": 21.6041649801129
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.415190040822669,
    "Emphasis on Accuracy 60:20:20": 4.72772947568733,
    "Emphasis on Cost 20:20:60": 7.305706070879352,
    "Emphasis on Speed 20:60:20": 8.386254186635064,
    "Intelligence only": 3.3901175711211273
   }
  },
  {
   "adapter_id": "images/kev-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 68.45285936403559,
    "cost": 65.78036497095258,
    "intelligence": 19.510707720813777,
    "speed": 90.22194613896767
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.18711800351703844,
      "ece_score": 62.576399296592314,
      "mean_tvd": null,
      "n": 733,
      "n_ece_hard": 306,
      "n_tvd": 0,
      "score": 62.576399296592314,
      "score_before_time_limit": 62.576399296592314,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.32884097035040427,
      "brier": 0.19162911051212952,
      "ece": 0.13239892183288407,
      "n": 371,
      "score": 73.52021563342319,
      "score_before_time_limit": 73.52021563342319,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8365122615803815,
      "interval90_width": 0.4604904632152589,
      "n": 367,
      "nrps": 0.15784808091493369,
      "score": 69.26196316209126,
      "score_before_time_limit": 69.26196316209126,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.22845632792162052
     }
    },
    "score": 68.45285936403559
   },
   "capability": 43.98178354242468,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     4.074342138455829,
     12.022833978108384
    ],
    "B": [
     3.184278465732148,
     10.032025050174324
    ],
    "C": [
     2.8294042628165483,
     8.349190262575268
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013824661330049262
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 4B (research preview)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 19.939346418532395,
    "I_sealed": 19.08206902309516,
    "base": 19.510707720813777,
    "excess": -1.7161102470010072,
    "gap": 0.8572773954372366,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 51.235731456250235,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 45.26580685750237,
       "judge": 44.62755438365195,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -4.347826086956519,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 8.582307799346953,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 34.24426078971534,
       "hard": 15.014273649587162,
       "judge": 21.314617902468203,
       "standard": -36.211065550999535
      }
     },
     "sealed|choice": {
      "cc": 42.547775845492595,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 56.10080555760846,
       "hard": 29.751131221719444,
       "judge": 44.134078212290504,
       "standard": 58.9850966867841
      }
     },
     "sealed|noul": {
      "cc": 5.208014566705289,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -30.097087378640776,
       "judge": 26.315789473684205,
       "standard": 26.7605633802817
      }
     },
     "sealed|score": {
      "cc": 9.490416657087591,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 60.8431474353109,
       "hard": -5.464015519199061,
       "judge": 4.345254884769856,
       "standard": 21.440658279025847
      }
     }
    }
   },
   "jevbench_score": 6.609647663198999,
   "key": "kev-4b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8152610441767069,
    "decisive_rate": 0.6711590296495957,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 68.45285936403559,
     "cost": 65.78036497095258,
     "intelligence": 19.510707720813777,
     "speed": 90.46585183268016
    },
    "board_release": "v1.6.43",
    "capability": 43.98178354242468,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 103,
    "capability_rank": 131,
    "composite_ci95": {
     "A": [
      4.075524448580174,
      12.02725721176505
     ],
     "B": [
      3.1848561369918413,
      10.0344884794078
     ],
     "C": [
      2.83022531151401,
      8.352261952614619
     ]
    },
    "jevbench_score": 6.611791819903415,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 142,
    "rank": 182,
    "ranks": {
     "A": 182,
     "B": 184,
     "C": 183,
     "capability": 192
    },
    "scores": {
     "A": 6.611791819903415,
     "B": 5.310194944940352,
     "C": 4.591522097155148
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29399301540106537,
     "p50_s_raw": 0.07199650770053267,
     "p95_s_adjusted": 0.3055468452628701,
     "p95_s_raw": 0.07777342263143508
    },
    "views": {
     "Balanced 33:33:33": 5.893456455385603,
     "Emphasis on Accuracy 60:20:20": 4.229224928104588,
     "Emphasis on Cost 20:20:60": 7.055012442959745,
     "Emphasis on Speed 20:60:20": 7.642572180991565,
     "Intelligence only": 2.970838616432075
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 203,
   "ranked": true,
   "ranks": {
    "A": 203,
    "B": 206,
    "C": 204,
    "capability": 213
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 6.609647663198999,
    "B": 5.309088373522138,
    "C": 4.590033099443748
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.29449559943750503,
    "p50_s_raw": 0.0722477997187525,
    "p95_s_adjusted": 0.3226462163031101,
    "p95_s_raw": 0.08632310815155504
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-4B-Base + LoRA + learned pointer head; jaredpalmer/kev-4b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.30814638480004,
      71.54002800358278
     ],
     "I": [
      16.228114704223106,
      24.71673328753193
     ],
     "capability": [
      39.998560456744215,
      47.239203316922335
     ],
     "composite": {
      "A": [
       4.074342138455829,
       12.022833978108384
      ],
      "B": [
       3.184278465732148,
       10.032025050174324
      ],
      "C": [
       2.8294042628165483,
       8.349190262575268
      ]
     },
     "seed": 54
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 68.45285936403559,
     "capability": 43.98178354242468,
     "intelligence": 19.510707720813777
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5693333333333334,
      "calibration": 62.576399296592314,
      "cc": 44.923884315527204,
      "cc_by_type": {
       "choice": 44.923884315527204
      },
      "cc_tiered": 46.891753650871415,
      "cc_tiered_open": 51.235731456250235,
      "cc_tiered_sealed": 42.547775845492595,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 44.923884315527204
     },
     "noul": {
      "accuracy": 0.5413333333333333,
      "calibration": 73.52021563342319,
      "cc": 8.266666666666666,
      "cc_by_type": {
       "noul": 8.266666666666666
      },
      "cc_tiered": 2.6040072833526446,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 5.208014566705289,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 8.266666666666666
     },
     "score": {
      "accuracy": 0.41333333333333333,
      "calibration": 69.26196316209126,
      "cc": 12.570612763030764,
      "cc_by_type": {
       "score": 12.570612763030764
      },
      "cc_tiered": 9.036362228217271,
      "cc_tiered_open": 8.582307799346953,
      "cc_tiered_sealed": 9.490416657087591,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 12.570612763030764
     }
    },
    "raw_on_A": {
     "calibration": 69.97372951951687,
     "capability": 47.498407356877294,
     "intelligence": 25.023085194237723
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.022666666666666613,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.021333333333333315,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 5.891185176143634,
    "Emphasis on Accuracy 60:20:20": 4.228522990501039,
    "Emphasis on Cost 20:20:60": 7.0530593435395375,
    "Emphasis on Speed 20:60:20": 7.635700567523155,
    "Intelligence only": 2.970838616432075
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 93.71720359195915,
    "cost": 18.302044660887503,
    "intelligence": 64.16071886717361,
    "speed": 82.8278891625431
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.034987296416937964,
      "ece_score": 93.0025407166124,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 93.0025407166124,
      "score_before_time_limit": 93.0025407166124,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.20485175202156336,
      "brier": 0.06491574711590294,
      "ece": 0.03041293800539084,
      "n": 371,
      "score": 93.91741239892183,
      "score_before_time_limit": 93.91741239892183,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.3089430894308942,
      "n": 369,
      "nrps": 0.049457903703703704,
      "score": 94.23165766034325,
      "score_before_time_limit": 94.23165766034325,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03295447154471563
     }
    },
    "score": 93.71720359195915
   },
   "capability": 78.93896122956639,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     5.6585469295985185,
     5.805864678191786
    ],
    "B": [
     6.0027759010783175,
     6.264807192340032
    ],
    "C": [
     5.477170313589639,
     5.805864678191786
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.52066024
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.5287680162491537
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Kev 27B (Kev 1.0, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 64.07452328888938,
    "I_sealed": 64.24691444545786,
    "base": 64.16071886717361,
    "excess": -2.745778799006724,
    "gap": -0.17239115656848014,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 81.24201973326763,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.23875432525952,
       "judge": 80.22412656558997,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 47.325051759834366,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 63.656498373566166,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.13878787878788,
       "hard": 65.75611111111111,
       "judge": 63.57169491525423,
       "standard": 57.843333333333334
      }
     },
     "sealed|choice": {
      "cc": 78.44659446797311,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 73.52941176470587,
       "judge": 80.44692737430168,
       "standard": 84.48084739499939
      }
     },
     "sealed|noul": {
      "cc": 50.836710400368496,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72.0,
       "hard": 33.98058252427185,
       "judge": 57.89473684210527,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 63.45743846803195,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 71.6254609929078,
       "hard": 54.08627450980392,
       "judge": 71.25748502994011,
       "standard": 66.4156852791878
      }
     }
    }
   },
   "jevbench_score": 5.764807634685908,
   "key": "kev-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "jaredpalmer/kev-27b@af0e6d551bdc2cc724f3e9d7a8bee1cd4fb8f7bf",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9694915254237289,
    "decisive_rate": 0.7951482479784366,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 93.71720359195915,
     "cost": 18.302044660887503,
     "intelligence": 64.16071886717361,
     "speed": 83.65684843680415
    },
    "board_release": "v1.6.43",
    "capability": 78.93896122956639,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      5.665703330300189,
      5.813398804963979
     ],
     "B": [
      6.009217533619645,
      6.271823804033908
     ],
     "C": [
      5.484111432603801,
      5.813398804963979
     ]
    },
    "jevbench_score": 5.77223551264444,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 144,
    "rank": 184,
    "ranks": {
     "A": 184,
     "B": 180,
     "C": 178,
     "capability": 29
    },
    "scores": {
     "A": 5.77223551264444,
     "B": 6.178201442095797,
     "C": 5.77223551264444
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5102782679954544,
     "p50_s_raw": 0.18013913399772719,
     "p95_s_adjusted": 0.8443219571985537,
     "p95_s_raw": 0.34716097859927686
    },
    "views": {
     "Balanced 33:33:33": 4.891297337690537,
     "Emphasis on Accuracy 60:20:20": 5.910281375617489,
     "Emphasis on Cost 20:20:60": 3.499139205379304,
     "Emphasis on Speed 20:60:20": 6.315006752900174,
     "Intelligence only": 8.596633940261478
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 205,
   "ranked": true,
   "ranks": {
    "A": 205,
    "B": 202,
    "C": 200,
    "capability": 34
   },
   "repo": "https://huggingface.co/jaredpalmer/kev-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 5.764807634685908,
    "B": 6.171392633488757,
    "C": 5.764807634685908
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.524460306018591,
    "p50_s_raw": 0.18723015300929546,
    "p95_s_adjusted": 0.9942565476871092,
    "p95_s_raw": 0.4221282738435546
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.38693398193061,
      92.90424439872974
     ],
     "I": [
      58.981544658104795,
      68.3141526180369
     ],
     "capability": [
      74.70682599002546,
      79.91812944044146
     ],
     "composite": {
      "A": [
       5.6585469295985185,
       5.805864678191786
      ],
      "B": [
       6.0027759010783175,
       6.264807192340032
      ],
      "C": [
       5.477170313589639,
       5.805864678191786
      ]
     },
     "seed": 62
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 93.71720359195915,
     "capability": 78.93896122956639,
     "intelligence": 64.16071886717361
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8386666666666667,
      "calibration": 93.0025407166124,
      "cc": 79.36777090457831,
      "cc_by_type": {
       "choice": 79.36777090457831
      },
      "cc_tiered": 79.84430710062037,
      "cc_tiered_open": 81.24201973326763,
      "cc_tiered_sealed": 78.44659446797311,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 79.36777090457831
     },
     "noul": {
      "accuracy": 0.7626666666666667,
      "calibration": 93.91741239892183,
      "cc": 52.533333333333346,
      "cc_by_type": {
       "noul": 52.533333333333346
      },
      "cc_tiered": 49.080881080101435,
      "cc_tiered_open": 47.325051759834366,
      "cc_tiered_sealed": 50.836710400368496,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 52.533333333333346
     },
     "score": {
      "accuracy": 0.704,
      "calibration": 94.23165766034325,
      "cc": 62.96492982456141,
      "cc_by_type": {
       "score": 62.96492982456141
      },
      "cc_tiered": 63.55696842079906,
      "cc_tiered_open": 63.656498373566166,
      "cc_tiered_sealed": 63.45743846803195,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.96492982456141
     }
    },
    "raw_on_A": {
     "calibration": 89.47604418949679,
     "capability": 78.40559149548258,
     "intelligence": 67.33513880146839
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 4.884186987672116,
    "Emphasis on Accuracy 60:20:20": 5.904049998028432,
    "Emphasis on Cost 20:20:60": 3.4969540766173703,
    "Emphasis on Speed 20:60:20": 6.293714188229336,
    "Intelligence only": 8.596633940261478
   }
  },
  {
   "adapter_id": "images/malkuth-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "newfull5 (dhtocks)",
   "axes": {
    "calibration": 79.34885520197874,
    "cost": 65.5661191189412,
    "intelligence": 15.917395195670291,
    "speed": 91.96529998043799
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1264719869706841,
      "ece_score": 74.70560260586319,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 74.70560260586319,
      "score_before_time_limit": 74.70560260586319,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.46630727762803237,
      "brier": 0.1896760988948787,
      "ece": 0.09166172506738543,
      "n": 371,
      "score": 81.6676549865229,
      "score_before_time_limit": 81.6676549865229,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9024390243902439,
      "interval90_width": 0.6144986449864495,
      "n": 369,
      "nrps": 0.14966934108401095,
      "score": 81.67330801355013,
      "score_before_time_limit": 81.67330801355013,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10843224932249321
     }
    },
    "score": 79.34885520197874
   },
   "capability": 47.633125198824516,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     2.189500800490991,
     6.223462431614387
    ],
    "B": [
     1.6395926065086215,
     4.929075458185204
    ],
    "C": [
     1.5204866670076322,
     4.321848910843324
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014053873152709357
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Malkuth-2B (newfull5, Kev post-train)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 16.94824627575026,
    "I_sealed": 14.88654411559032,
    "base": 15.917395195670291,
    "excess": -0.511685482278303,
    "gap": 2.061702160159941,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 24.95619850881217,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 16.955017301038072,
       "judge": 28.80685563612393,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": -21.739130434782606,
       "judge": -40.0,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 25.88854031843862,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.77636363636364,
       "hard": 12.745185185185182,
       "judge": 28.807457627118627,
       "standard": 35.852962962962955
      }
     },
     "sealed|choice": {
      "cc": 39.92521849097588,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 50.77969107974283,
       "hard": 26.18778280542985,
       "judge": 44.134078212290504,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -39.80582524271845,
       "judge": -23.684210526315784,
       "standard": 18.309859154929576
      }
     },
     "sealed|score": {
      "cc": 4.734413855795083,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 40.8195035460993,
       "hard": -11.993921568627375,
       "judge": 11.762814371257434,
       "standard": 9.605939086294368
      }
     }
    }
   },
   "jevbench_score": 3.9918472005413665,
   "key": "malkuth-2b",
   "last_measured_on": "2026-10-02",
   "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8434343434343434,
    "decisive_rate": 0.5336927223719676,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.34885520197874,
     "cost": 65.5661191189412,
     "intelligence": 15.917395195670291,
     "speed": 92.03462875001455
    },
    "board_release": "v1.6.43",
    "capability": 47.633125198824516,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 96,
    "capability_rank": 123,
    "composite_ci95": {
     "A": [
      2.1896531510798423,
      6.224018858922979
     ],
     "B": [
      1.639660951009494,
      4.9293547200832295
     ],
     "C": [
      1.5205924660276677,
      4.322235318696512
     ]
    },
    "jevbench_score": 3.992169201455145,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 146,
    "rank": 186,
    "ranks": {
     "A": 186,
     "B": 189,
     "C": 187,
     "capability": 182
    },
    "scores": {
     "A": 3.992169201455145,
     "B": 3.08287077876474,
     "C": 2.7723397232327396
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23610859100008383,
     "p50_s_raw": 0.04305429550004192,
     "p95_s_adjusted": 0.26510952737880866,
     "p95_s_raw": 0.05755476368940435
    },
    "views": {
     "Balanced 33:33:33": 3.4183795643165196,
     "Emphasis on Accuracy 60:20:20": 2.361369226233909,
     "Emphasis on Cost 20:20:60": 4.242339522263916,
     "Emphasis on Speed 20:60:20": 4.578614246342653,
     "Intelligence only": 1.6131545908775944
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 209,
   "ranked": true,
   "ranks": {
    "A": 209,
    "B": 212,
    "C": 210,
    "capability": 203
   },
   "repo": "https://github.com/newfull5/malkuth",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.9918472005413665,
    "B": 3.0827171568255443,
    "C": 2.77211611148706
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23758355304598808,
    "p50_s_raw": 0.04379177652299404,
    "p95_s_adjusted": 0.26770324098761195,
    "p95_s_raw": 0.05885162049380597
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.7053529716356,
      81.31914491234029
     ],
     "I": [
      12.692506481394203,
      18.87608002604669
     ],
     "capability": [
      43.451731383062054,
      49.395128138793496
     ],
     "composite": {
      "A": [
       2.189500800490991,
       6.223462431614387
      ],
      "B": [
       1.6395926065086215,
       4.929075458185204
      ],
      "C": [
       1.5204866670076322,
       4.321848910843324
      ]
     },
     "seed": 63
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.34885520197874,
     "capability": 47.633125198824516,
     "intelligence": 15.917395195670291
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5133333333333333,
      "calibration": 74.70560260586319,
      "cc": 37.76228413364529,
      "cc_by_type": {
       "choice": 37.76228413364529
      },
      "cc_tiered": 32.44070849989403,
      "cc_tiered_open": 24.95619850881217,
      "cc_tiered_sealed": 39.92521849097588,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.76228413364529
     },
     "noul": {
      "accuracy": 0.44533333333333336,
      "calibration": 81.6676549865229,
      "cc": -10.933333333333328,
      "cc_by_type": {
       "noul": -10.933333333333328
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.38133333333333336,
      "calibration": 81.67330801355013,
      "cc": 9.127684210526233,
      "cc_by_type": {
       "score": 9.127684210526233
      },
      "cc_tiered": 15.31147708711685,
      "cc_tiered_open": 25.88854031843862,
      "cc_tiered_sealed": 4.734413855795083,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 9.127684210526233
     }
    },
    "raw_on_A": {
     "calibration": 76.45071634453062,
     "capability": 47.2825063700716,
     "intelligence": 18.114296395612577
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.4180647796533634,
    "Emphasis on Accuracy 60:20:20": 2.3612790949695106,
    "Emphasis on Cost 20:20:60": 4.242048621201638,
    "Emphasis on Speed 20:60:20": 4.577597863816472,
    "Intelligence only": 1.6131545908775944
   }
  },
  {
   "adapter_id": "images/qwen3-reranker-4b/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Qwen",
   "axes": {
    "calibration": 73.15350658389404,
    "cost": 48.39770052432463,
    "intelligence": 16.337944014530347,
    "speed": 81.05912758410997
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12818639012273808,
      "ece_score": 74.36272197545239,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 74.36272197545239,
      "score_before_time_limit": 74.36272197545239,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.7466666666666666,
      "brier": 0.23186747724503004,
      "ece": 0.1932172851672485,
      "n": 375,
      "score": 61.3565429665503,
      "score_before_time_limit": 61.3565429665503,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9093333333333333,
      "interval90_width": 0.6708888888888882,
      "n": 375,
      "nrps": 0.15088039394905625,
      "score": 83.74125480967948,
      "score_before_time_limit": 83.74125480967948,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.08714725492867716
     }
    },
    "score": 73.15350658389404
   },
   "capability": 44.7457252992122,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     2.2247087486067683,
     5.414794488782484
    ],
    "B": [
     1.7217460597182237,
     4.4175289830398015
    ],
    "C": [
     1.544936630976922,
     3.760273950543392
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: hosted exact-model reference (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05249000923645321
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Qwen3-Reranker-4B",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 17.351796606180663,
    "I_sealed": 15.324091422880029,
    "base": 16.337944014530347,
    "excess": -0.5456824591376095,
    "gap": 2.0277051833006343,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 36.57581104234733,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 22.617175212330924,
       "judge": 52.537903757415954,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -91.30434782608697,
       "judge": -40.0,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 15.479578776194659,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 16.665491764012373,
       "hard": -1.4208863867574584,
       "judge": 33.97314797019466,
       "standard": 20.94719881719004
      }
     },
     "sealed|choice": {
      "cc": 33.418936417285686,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 45.458576601877176,
       "hard": 22.115384615384603,
       "judge": 34.823091247672274,
       "standard": 47.89998768321223
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -43.99999999999999,
       "hard": -84.46601941747574,
       "judge": -57.89473684210527,
       "standard": -43.66197183098591
      }
     },
     "sealed|score": {
      "cc": 12.553337851354403,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 16.14176889876734,
       "hard": 9.253841484765735,
       "judge": 15.752141635882,
       "standard": 12.559909384033885
      }
     }
    }
   },
   "jevbench_score": 3.7093667042558325,
   "key": "qwen3-reranker-4b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7473684210526316,
    "decisive_rate": 0.25333333333333335,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.15350658389404,
     "cost": 48.39770052432463,
     "intelligence": 16.337944014530347,
     "speed": 82.14971164581814
    },
    "board_release": "v1.6.43",
    "capability": 44.7457252992122,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 100,
    "capability_rank": 128,
    "composite_ci95": {
     "A": [
      2.2277188584602894,
      5.423744955848025
     ],
     "B": [
      1.7231876454208677,
      4.422291990633802
     ],
     "C": [
      1.5470269850418672,
      3.766489552672239
     ]
    },
    "jevbench_score": 3.715006798009075,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 147,
    "rank": 187,
    "ranks": {
     "A": 187,
     "B": 190,
     "C": 188,
     "capability": 186
    },
    "scores": {
     "A": 3.715006798009075,
     "B": 2.961112195421367,
     "C": 2.5798658319507455
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5581826242618263,
     "p50_s_raw": 0.20409131213091314,
     "p95_s_adjusted": 1.092075143335387,
     "p95_s_raw": 0.47103757166769356
    },
    "views": {
     "Balanced 33:33:33": 3.1912622823260945,
     "Emphasis on Accuracy 60:20:20": 2.310804869814862,
     "Emphasis on Cost 20:20:60": 3.695072544212985,
     "Emphasis on Speed 20:60:20": 4.224997058133206,
     "Intelligence only": 1.6344124017617998
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 210,
   "ranked": true,
   "ranks": {
    "A": 210,
    "B": 213,
    "C": 211,
    "capability": 209
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-Reranker-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.7093667042558325,
    "B": 2.958244019495998,
    "C": 2.5759491001776604
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5727677877061069,
    "p50_s_raw": 0.21138389385305345,
    "p95_s_adjusted": 1.3680710606742645,
    "p95_s_raw": 0.6090355303371323
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "4B instruction-aware generative reranker",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.92034254293053,
      76.98495141643136
     ],
     "I": [
      13.411885193349915,
      18.933037790738602
     ],
     "capability": [
      41.28670365010265,
      47.260558123220996
     ],
     "composite": {
      "A": [
       2.2247087486067683,
       5.414794488782484
      ],
      "B": [
       1.7217460597182237,
       4.4175289830398015
      ],
      "C": [
       1.544936630976922,
       3.760273950543392
      ]
     },
     "seed": 91
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.15350658389404,
     "capability": 44.7457252992122,
     "intelligence": 16.337944014530347
    },
    "per_type": {
     "choice": {
      "accuracy": 0.4866666666666667,
      "calibration": 74.36272197545239,
      "cc": 34.35199833274914,
      "cc_by_type": {
       "choice": 34.35199833274914
      },
      "cc_tiered": 34.99737372981651,
      "cc_tiered_open": 36.57581104234733,
      "cc_tiered_sealed": 33.418936417285686,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 34.35199833274914
     },
     "noul": {
      "accuracy": 0.18933333333333333,
      "calibration": 61.3565429665503,
      "cc": -62.133333333333326,
      "cc_by_type": {
       "noul": -62.133333333333326
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.37333333333333335,
      "calibration": 83.74125480967948,
      "cc": 12.388563279917397,
      "cc_by_type": {
       "score": 12.388563279917397
      },
      "cc_tiered": 14.016458313774532,
      "cc_tiered_open": 15.479578776194659,
      "cc_tiered_sealed": 12.553337851354403,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 12.388563279917397
     }
    },
    "raw_on_A": {
     "calibration": 68.75797234609402,
     "capability": 42.734177704587836,
     "intelligence": 16.71038306308165
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.1857143008545523,
    "Emphasis on Accuracy 60:20:20": 2.309057780921745,
    "Emphasis on Cost 20:20:60": 3.6906073768403598,
    "Emphasis on Speed 20:60:20": 4.207535124081076,
    "Intelligence only": 1.6344124017617998
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Featherless",
   "axes": {
    "calibration": 91.85610710600702,
    "cost": 15.10193161538902,
    "intelligence": 57.92931033189586,
    "speed": 75.96701882114468
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0559935150394999,
      "ece_score": 88.80129699210002,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.80129699210002,
      "score_before_time_limit": 88.80129699210002,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.21832884097035044,
      "brier": 0.07396200744195928,
      "ece": 0.02357869016917279,
      "n": 371,
      "score": 95.28426196616545,
      "score_before_time_limit": 95.28426196616545,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9701897018970189,
      "interval90_width": 0.2899728997289972,
      "n": 369,
      "nrps": 0.061950920450199856,
      "score": 91.48276235975561,
      "score_before_time_limit": 91.48276235975561,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05419691617734392
     }
    },
    "score": 91.85610710600702
   },
   "capability": 74.89270871895144,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     3.3234512116257737,
     3.4279484763998758
    ],
    "B": [
     3.5395885701607335,
     3.7233733360846806
    ],
    "C": [
     2.54666158897383,
     3.4279484763998758
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.66561768
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.6759827488151658
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SimpleJev Qwen3.8-27B (self-hosted, v1.6 pool)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 58.77016996644363,
    "I_sealed": 57.08845069734809,
    "base": 57.92931033189586,
    "excess": -0.8916683733426987,
    "gap": 1.6817192690955451,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.38232847559632,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 84.17930125247197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 39.7632850241546,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 57.16489639957998,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.0815846586252,
       "hard": 50.99462651293531,
       "judge": 64.6276543190174,
       "standard": 50.35295516419056
      }
     },
     "sealed|choice": {
      "cc": 67.17049093981976,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 61.312217194570124,
       "judge": 64.61824953445065,
       "standard": 78.93829289321346
      }
     },
     "sealed|noul": {
      "cc": 46.31434719713273,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68.0,
       "hard": 26.21359223300972,
       "judge": 52.63157894736843,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 57.78051395509178,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.9707943343972,
       "hard": 48.80809948321188,
       "judge": 65.15163618443506,
       "standard": 58.0735193651839
      }
     }
    }
   },
   "jevbench_score": 3.3935801379641406,
   "key": "simplejev-qwen3.8-27b-selfhost",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 + featherless-ai/simple-jev@9c11582d3e631e0b118f3f3132ebbb00c62020c5",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9620689655172414,
    "decisive_rate": 0.7816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.85610710600702,
     "cost": 15.10193161538902,
     "intelligence": 57.92931033189586,
     "speed": 76.19964460590472
    },
    "board_release": "v1.6.43",
    "capability": 74.89270871895144,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      3.3246680487142375,
      3.4292430519298938
     ],
     "B": [
      3.5406927144972373,
      3.7245951371091817
     ],
     "C": [
      2.5475929721327564,
      3.4292430519298938
     ]
    },
    "jevbench_score": 3.394848880196433,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 149,
    "rank": 189,
    "ranks": {
     "A": 189,
     "B": 185,
     "C": 185,
     "capability": 45
    },
    "scores": {
     "A": 3.394848880196433,
     "B": 3.656360936982179,
     "C": 3.164569675341542
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.393547774007311,
     "p50_s_raw": 0.6217738870036555,
     "p95_s_adjusted": 1.721526361943688,
     "p95_s_raw": 0.785763180971844
    },
    "views": {
     "Balanced 33:33:33": 2.833072735614859,
     "Emphasis on Accuracy 60:20:20": 3.4785771436394777,
     "Emphasis on Cost 20:20:60": 1.9915507939259591,
     "Emphasis on Speed 20:60:20": 3.7129751262531117,
     "Intelligence only": 5.284736623506664
    }
   },
   "provenance": {
    "carried_metadata_from": "author repository + base config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 211,
   "ranked": true,
   "ranks": {
    "A": 211,
    "B": 208,
    "C": 208,
    "capability": 57
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.3935801379641406,
    "B": 3.655183485364579,
    "C": 3.1633869943634405
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.42801156102214,
    "p50_s_raw": 0.63900578051107,
    "p95_s_adjusted": 1.7724189897620815,
    "p95_s_raw": 0.8112094948810408
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.56600011746222,
      91.91458842585743
     ],
     "I": [
      52.49374537062954,
      62.50608217507831
     ],
     "capability": [
      70.32359564579012,
      76.5602273459949
     ],
     "composite": {
      "A": [
       3.3234512116257737,
       3.4279484763998758
      ],
      "B": [
       3.5395885701607335,
       3.7233733360846806
      ],
      "C": [
       2.54666158897383,
       3.4279484763998758
      ]
     },
     "seed": 112
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.85610710600702,
     "capability": 74.89270871895144,
     "intelligence": 57.92931033189586
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 88.80129699210002,
      "cc": 69.98948495211388,
      "cc_by_type": {
       "choice": 69.98948495211388
      },
      "cc_tiered": 73.27640970770804,
      "cc_tiered_open": 79.38232847559632,
      "cc_tiered_sealed": 67.17049093981976,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.98948495211388
     },
     "noul": {
      "accuracy": 0.744,
      "calibration": 95.28426196616545,
      "cc": 48.8,
      "cc_by_type": {
       "noul": 48.8
      },
      "cc_tiered": 43.038816110643666,
      "cc_tiered_open": 39.7632850241546,
      "cc_tiered_sealed": 46.31434719713273,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 48.8
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 91.48276235975561,
      "cc": 57.30970745563964,
      "cc_by_type": {
       "score": 57.30970745563964
      },
      "cc_tiered": 57.47270517733588,
      "cc_tiered_open": 57.16489639957998,
      "cc_tiered_sealed": 57.78051395509178,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.30970745563964
     }
    },
    "raw_on_A": {
     "calibration": 90.15198763991802,
     "capability": 76.03520978092624,
     "intelligence": 61.91843192193445
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.8318946729348573,
    "Emphasis on Accuracy 60:20:20": 3.477511394408173,
    "Emphasis on Cost 20:20:60": 1.9912014192616554,
    "Emphasis on Speed 20:60:20": 3.709334935251932,
    "Intelligence only": 5.284736623506664
   }
  },
  {
   "adapter_id": "images/zerank-2/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ZeroEntropy",
   "axes": {
    "calibration": 83.20040692874471,
    "cost": 48.39770052432463,
    "intelligence": 15.596611891228248,
    "speed": 81.79510880020156
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09594830772201698,
      "ece_score": 80.8103384555966,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 80.8103384555966,
      "score_before_time_limit": 80.8103384555966,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.8613333333333333,
      "brier": 0.2189278600834461,
      "ece": 0.0929386859803417,
      "n": 375,
      "score": 81.41226280393165,
      "score_before_time_limit": 81.41226280393165,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.9126666666666668,
      "n": 375,
      "nrps": 0.14987040001080157,
      "score": 87.3786195267059,
      "score_before_time_limit": 87.3786195267059,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05127860472754006
     }
    },
    "score": 83.20040692874471
   },
   "capability": 49.39850940998648,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     1.9726818243303679,
     4.836385111035285
    ],
    "B": [
     1.5059161012419389,
     3.8995465309772963
    ],
    "C": [
     1.3699179335627552,
     3.3586007715522808
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05249000923645321
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ZeroEntropy zerank-2",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 15.496935616681888,
    "I_sealed": 15.696288165774607,
    "base": 15.596611891228248,
    "excess": -2.772740191530963,
    "gap": -0.19935254909271904,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 36.28749068653345,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 49.40540540540541,
       "hard": 30.16671909405474,
       "judge": 32.76203032300594,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -82.6086956521739,
       "judge": -70.0,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 10.203316163512213,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 13.133808997075057,
       "hard": 3.6957598546269232,
       "judge": 27.616853766877792,
       "standard": -4.367124040546999
      }
     },
     "sealed|choice": {
      "cc": 34.99399091747191,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 50.77969107974283,
       "hard": 27.205882352941163,
       "judge": 36.68528864059592,
       "standard": 40.14041138071193
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -88.0,
       "hard": -86.40776699029126,
       "judge": -71.05263157894737,
       "standard": -52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 12.094873579851914,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 16.780298372211313,
       "hard": 6.472246249261849,
       "judge": 17.43542452331028,
       "standard": 12.986589429664797
      }
     }
    }
   },
   "jevbench_score": 3.3447985015536363,
   "key": "zerank-2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8653846153846154,
    "decisive_rate": 0.13866666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.20040692874471,
     "cost": 48.39770052432463,
     "intelligence": 15.596611891228248,
     "speed": 82.89783918855927
    },
    "board_release": "v1.6.43",
    "capability": 49.39850940998648,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 92,
    "capability_rank": 119,
    "composite_ci95": {
     "A": [
      1.9752898202498443,
      4.84420234615273
     ],
     "B": [
      1.507128562340162,
      3.903609846255529
     ],
     "C": [
      1.3717290418401693,
      3.364029407050507
     ]
    },
    "jevbench_score": 3.349795360318651,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 150,
    "rank": 190,
    "ranks": {
     "A": 190,
     "B": 193,
     "C": 191,
     "capability": 176
    },
    "scores": {
     "A": 3.349795360318651,
     "B": 2.635183024414276,
     "C": 2.3262467779990628
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4991468541324139,
     "p50_s_raw": 0.17457342706620693,
     "p95_s_adjusted": 1.0279872846323996,
     "p95_s_raw": 0.4389936423161998
    },
    "views": {
     "Balanced 33:33:33": 2.824157802929915,
     "Emphasis on Accuracy 60:20:20": 2.025225537167865,
     "Emphasis on Cost 20:20:60": 3.299126425382793,
     "Emphasis on Speed 20:60:20": 3.7681689882350637,
     "Intelligence only": 1.4218711229563352
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 212,
   "ranked": true,
   "ranks": {
    "A": 212,
    "B": 215,
    "C": 213,
    "capability": 196
   },
   "repo": "https://huggingface.co/zeroentropy/zerank-2-reranker",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.3447985015536363,
    "B": 2.632707813159186,
    "C": 2.3227767371900248
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5193217188119889,
    "p50_s_raw": 0.18466085940599442,
    "p95_s_adjusted": 1.2736574321053922,
    "p95_s_raw": 0.5618287160526961
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-Reranker-derived 4B cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.23040686190234,
      86.00830164706042
     ],
     "I": [
      12.746131677720898,
      18.039638464222975
     ],
     "capability": [
      46.43325156721317,
      50.75210427056033
     ],
     "composite": {
      "A": [
       1.9726818243303679,
       4.836385111035285
      ],
      "B": [
       1.5059161012419389,
       3.8995465309772963
      ],
      "C": [
       1.3699179335627552,
       3.3586007715522808
      ]
     },
     "seed": 110
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.20040692874471,
     "capability": 49.39850940998648,
     "intelligence": 15.596611891228248
    },
    "per_type": {
     "choice": {
      "accuracy": 0.49866666666666665,
      "calibration": 80.8103384555966,
      "cc": 35.8866269431524,
      "cc_by_type": {
       "choice": 35.8866269431524
      },
      "cc_tiered": 35.64074080200268,
      "cc_tiered_open": 36.28749068653345,
      "cc_tiered_sealed": 34.99399091747191,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 35.8866269431524
     },
     "noul": {
      "accuracy": 0.12,
      "calibration": 81.41226280393165,
      "cc": -76.0,
      "cc_by_type": {
       "noul": -76.0
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.34933333333333333,
      "calibration": 87.3786195267059,
      "cc": 11.246326611904923,
      "cc_by_type": {
       "score": 11.246326611904923
      },
      "cc_tiered": 11.149094871682063,
      "cc_tiered_open": 10.203316163512213,
      "cc_tiered_sealed": 12.094873579851914,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 11.246326611904923
     }
    },
    "raw_on_A": {
     "calibration": 80.48954444569493,
     "capability": 48.704671856760854,
     "intelligence": 16.919799267826768
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.819423055893096,
    "Emphasis on Accuracy 60:20:20": 2.0237632525033584,
    "Emphasis on Cost 20:20:60": 3.2952477283993304,
    "Emphasis on Speed 20:60:20": 3.753032234510616,
    "Intelligence only": 1.4218711229563352
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Dongmin Kim / @carrtesy (tommy.dm.kim@gmail.com) - unofficial personal project",
   "axes": {
    "calibration": 80.52861474872404,
    "cost": 57.919479713067595,
    "intelligence": 14.831288392065307,
    "speed": 93.81713750966469
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1631615191239309,
      "ece_score": 67.36769617521382,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 67.36769617521382,
      "score_before_time_limit": 67.36769617521382,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.5983827493261455,
      "brier": 0.1968188345782441,
      "ece": 0.06151585075654434,
      "n": 371,
      "score": 87.69682984869114,
      "score_before_time_limit": 87.69682984869114,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.948509485094851,
      "interval90_width": 0.7529358626919599,
      "n": 369,
      "nrps": 0.15012627140838788,
      "score": 86.52131822226714,
      "score_before_time_limit": 86.52131822226714,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05972368207313468
     }
    },
    "score": 80.52861474872404
   },
   "capability": 47.67995157039467,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     1.695484672747176,
     5.268678060115385
    ],
    "B": [
     1.2625009050434257,
     4.159710667472831
    ],
    "C": [
     1.1774199116299833,
     3.6588042084134624
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no bookable listing for this base). THE RATE IS RUN 48'S, CARRIED UNCHANGED FOR THE SAME BASE: run 48 priced exaone-4.0-1.2b-jev (v0.2, identical base) at USD 0.02 per 1M input and 0 output, as the HIGHER of the two bracketing published rates, chosen in that direction because the rate was fixed after the measurement existed. Keeping it makes v0.3 directly comparable with v0.2 and takes no new decision. The frozen 25 Sep snapshot still contains NO listing for LGAI-EXAONE/EXAONE-4.0-1.2B (pricing_v15.reference() raises: 'base model lacks an explicit market-listing decision'), and still brackets 1.2B between deepinfra:Qwen/Qwen3.5-0.8B at (0.01, 0.05) and deepinfra:Qwen/Qwen3.5-2B at (0.02, 0.10). Estimated, not charged. RELEASE-LANE LINE, recorded and not acted on, AND IT IS RUN 48'S CARRIED LINE, NOT A NEW ONE: LGAI-EXAONE/EXAONE-4.0-1.2B must be adopted into the frozen UNLISTED_BASE_MODELS table before either EXAONE row may be published. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.02488704
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025274583615436694
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "EXAONE-4.0-1.2B-JEV v0.3 (unofficial fine-tune of LGAI-EXAONE/EXAONE-4.0-1.2B, native option-label softmax, two reads averaged, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,477 of 1,500 answered. NEW REVISION of the row run 48 measured: weights carrtesy/EXAONE-4.0-1.2B-JEV at TAG v0.3, server github.com/carrtesy/EXAONE-JEV at tag v0.3 (commit recorded in receipts/POD-R58.json), both the artefacts the author's own submit-form entry names. EXAONE AI Model License 1.2 - NC, NON-COMMERCIAL, and an unofficial personal project - the author states it is 'not an LG AI Research release'; both belong on the row. NO MAPPING DECISION OF OURS: their serve/systemone_server.py speaks the TypeSafe Jev POST /v1/systemone contract, so the UNCHANGED jevbench.adapters.typesafe emits exactly what it parses, and all eight checks of the unamended smoke gate passed on synthetic requests before any sealed item was sent. Read count (permute 2: options in the given and the reversed order, distributions averaged) and the per-type temperatures come from the weights' own jev_config.json, which the author's doc states needs no flags - their shipped default, not a setting of ours. STACK: the author's own docs/REPRODUCE.md install line in an isolated venv, vllm==0.30.0 letting it resolve its own stack (torch 2.13.0+cu130, transformers 5.19.0), served with their own published command verbatim (`python serve/systemone_server.py --model weights --port 8000 --gpu-util 0.9`) and VLLM_USE_FLASHINFER_SAMPLER=0 as their doc sets it. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM, at the limit the author DECLARES: their docs/REPRODUCE.md section 3 states a 65,536-token prompt limit answering 400, and all 23 came back as 'HTTP 400: prompt of 84996 tokens is longer than the maximum context length 65536'. The 23 invalid are therefore the system's own disclosed refusal, counted wrong, and no other item failed. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 58, benchmarkheaven.com/submit ref CA53D793 (bh-submit 5631d258-1eaa-4e2d-a658-f6417b85b351); previously GitHub issue #176",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090 32 GB",
   "intelligence": {
    "I_open": 16.786521241837672,
    "I_sealed": 12.87605554229294,
    "base": 14.831288392065307,
    "excess": 1.337078057106488,
    "gap": 3.9104656995447318,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 39.40654352431394,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 49.40540540540541,
       "hard": 20.729789241899976,
       "judge": 40.672379696769944,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -47.82608695652174,
       "judge": -40.0,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 10.953020201199083,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.5784737243812,
       "hard": -2.3251786244617323,
       "judge": 27.73524201685903,
       "standard": -0.4766416325602707
      }
     },
     "sealed|choice": {
      "cc": 28.792593942604675,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 45.458576601877176,
       "hard": 18.04298642533935,
       "judge": 32.96089385474861,
       "standard": 35.70636777928318
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 15.999999999999993,
       "hard": -63.106796116504846,
       "judge": -42.10526315789473,
       "standard": -9.859154929577462
      }
     },
     "sealed|score": {
      "cc": 9.835572684274151,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 36.004469283378306,
       "hard": -2.325764560565169,
       "judge": 14.49323221885701,
       "standard": 14.08730957252643
      }
     }
    }
   },
   "jevbench_score": 3.265807063300941,
   "key": "exaone-4.0-1.2b-jev-v0.3",
   "last_measured_on": "2026-10-07",
   "licence": "EXAONE AI Model License 1.2 (non-commercial)",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "carrtesy/EXAONE-4.0-1.2B-JEV@v0.3",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8456375838926175,
    "decisive_rate": 0.40161725067385445,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 80.52861474872404,
     "cost": 57.919479713067595,
     "intelligence": 14.831288392065307,
     "speed": 94.09940251974388
    },
    "board_release": "v1.6.43",
    "capability": 47.67995157039467,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 95,
    "capability_rank": 122,
    "composite_ci95": {
     "A": [
      1.6959116121891469,
      5.270428159114469
     ],
     "B": [
      1.262690264794709,
      4.160583281665687
     ],
     "C": [
      1.1777163973535743,
      3.660019554940604
     ]
    },
    "jevbench_score": 3.266776278413127,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 151,
    "rank": 191,
    "ranks": {
     "A": 191,
     "B": 194,
     "C": 192,
     "capability": 181
    },
    "scores": {
     "A": 3.266776278413127,
     "B": 2.5116077374172012,
     "C": 2.2685946377868937
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19053296000347472,
     "p50_s_raw": 0.020266480001737364,
     "p95_s_adjusted": 0.20421593803039284,
     "p95_s_raw": 0.02710796901519643
    },
    "views": {
     "Balanced 33:33:33": 2.7692786828242713,
     "Emphasis on Accuracy 60:20:20": 1.9113656734726492,
     "Emphasis on Cost 20:20:60": 3.388067484365469,
     "Emphasis on Speed 20:60:20": 3.7739415394848543,
     "Intelligence only": 1.304958289921029
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 213,
   "ranked": true,
   "ranks": {
    "A": 213,
    "B": 216,
    "C": 214,
    "capability": 202
   },
   "repo": "https://huggingface.co/carrtesy/EXAONE-4.0-1.2B-JEV",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.265807063300941,
    "B": 2.5111493585324483,
    "C": 2.267921571736765
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19166760700172744,
    "p50_s_raw": 0.020833803500863723,
    "p95_s_adjusted": 0.21663944199390245,
    "p95_s_raw": 0.03331972099695122
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "LGAI-EXAONE/EXAONE-4.0-1.2B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      74.05366116439995,
      82.35803770559488
     ],
     "I": [
      11.603293028025588,
      17.810851432602874
     ],
     "capability": [
      43.59835541237688,
      49.150688153062475
     ],
     "composite": {
      "A": [
       1.695484672747176,
       5.268678060115385
      ],
      "B": [
       1.2625009050434257,
       4.159710667472831
      ],
      "C": [
       1.1774199116299833,
       3.6588042084134624
      ]
     },
     "seed": 53
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.52861474872404,
     "capability": 47.67995157039467,
     "intelligence": 14.831288392065307
    },
    "per_type": {
     "choice": {
      "accuracy": 0.45866666666666667,
      "calibration": 67.36769617521382,
      "cc": 30.771198241808182,
      "cc_by_type": {
       "choice": 30.771198241808182
      },
      "cc_tiered": 34.09956873345931,
      "cc_tiered_open": 39.40654352431394,
      "cc_tiered_sealed": 28.792593942604675,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 30.771198241808182
     },
     "noul": {
      "accuracy": 0.336,
      "calibration": 87.69682984869114,
      "cc": -32.8,
      "cc_by_type": {
       "noul": -32.8
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.3893333333333333,
      "calibration": 86.52131822226714,
      "cc": 10.441024816632405,
      "cc_by_type": {
       "score": 10.441024816632405
      },
      "cc_tiered": 10.394296442736618,
      "cc_tiered_open": 10.953020201199083,
      "cc_tiered_sealed": 9.835572684274151,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 10.441024816632405
     }
    },
    "raw_on_A": {
     "calibration": 76.1279082946144,
     "capability": 45.86026412407656,
     "intelligence": 15.592619953538719
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.7683500657470317,
    "Emphasis on Accuracy 60:20:20": 1.9111001961152008,
    "Emphasis on Cost 20:20:60": 3.387233424113146,
    "Emphasis on Speed 20:60:20": 3.7708387334363804,
    "Intelligence only": 1.304958289921029
   }
  },
  {
   "adapter_id": "images/open-jev-zefan-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zefan Cai (@Zefan_Cai)",
   "axes": {
    "calibration": 73.61830158128652,
    "cost": 33.05490478272432,
    "intelligence": 21.804441188912403,
    "speed": 77.13082993864059
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1271563639749637,
      "ece_score": 74.56872720500726,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 74.56872720500726,
      "score_before_time_limit": 74.56872720500726,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3261455525606469,
      "brier": 0.22536579202900195,
      "ece": 0.1725448435767295,
      "n": 371,
      "score": 65.4910312846541,
      "score_before_time_limit": 65.4910312846541,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9322493224932249,
      "interval90_width": 0.6887985546522127,
      "n": 369,
      "nrps": 0.14871270327718153,
      "score": 80.79514625419819,
      "score_before_time_limit": 80.79514625419819,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.11769218581942746
     }
    },
    "score": 73.61830158128652
   },
   "capability": 47.71137138509946,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     1.6444221674835224,
     4.56371492730698
    ],
    "B": [
     1.3617013504482853,
     4.047573981646406
    ],
    "C": [
     1.141959838530224,
     3.1692464772965137
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.17041317733990147
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open-Jev 2B (Zefan Cai)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 27.422707291556733,
    "I_sealed": 16.38819706190457,
    "base": 21.905452176730652,
    "excess": 8.46112258721392,
    "gap": 11.034510229652163,
    "penalty": 0.9953887741278608,
    "per_type_split": {
     "open|choice": {
      "cc": 41.30120233813892,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.837837837837846,
       "hard": 26.39194715319283,
       "judge": 44.62755438365195,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 17.126293995859218,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 23.84062554067206,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.74006952878567,
       "hard": 15.775595079225702,
       "judge": 40.5382915597717,
       "standard": 8.974465440858504
      }
     },
     "sealed|choice": {
      "cc": 35.268479371367654,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 41.46774074347795,
       "hard": 27.714932126696816,
       "judge": 38.54748603351957,
       "standard": 42.35743318142629
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 15.999999999999993,
       "hard": -28.155339805825243,
       "judge": -5.263157894736848,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 13.89611181434606,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 33.49212867759059,
       "hard": -0.08856264182495188,
       "judge": 18.858024093873603,
       "standard": 24.62458387577451
      }
     }
    }
   },
   "jevbench_score": 3.238364668034295,
   "key": "open-jev-zefan-2b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.748,
    "decisive_rate": 0.6738544474393531,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.61830158128652,
     "cost": 33.05490478272432,
     "intelligence": 21.804441188912403,
     "speed": 77.13731150142542
    },
    "board_release": "v1.6.43",
    "capability": 47.71137138509946,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      1.6444373649451645,
      4.563766368468016
     ],
     "B": [
      1.361709687197464,
      4.0476062027996145
     ],
     "C": [
      1.1419703923230309,
      3.169282200325012
     ]
    },
    "jevbench_score": 3.2383990318693976,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 152,
    "rank": 192,
    "ranks": {
     "A": 192,
     "B": 192,
     "C": 193,
     "capability": 180
    },
    "scores": {
     "A": 3.2383990318693976,
     "B": 2.798036205873629,
     "C": 2.248888216575971
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.0230181839782744,
     "p50_s_raw": 0.43650909198913723,
     "p95_s_adjusted": 1.8896679471712559,
     "p95_s_raw": 0.869833973585628
    },
    "views": {
     "Balanced 33:33:33": 2.799165550074912,
     "Emphasis on Accuracy 60:20:20": 2.2985053544500897,
     "Emphasis on Cost 20:20:60": 2.7782159796690187,
     "Emphasis on Speed 20:60:20": 3.6135076182350976,
     "Intelligence only": 1.812285794889051
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 214,
   "ranked": true,
   "ranks": {
    "A": 214,
    "B": 214,
    "C": 215,
    "capability": 201
   },
   "repo": "https://github.com/Zefan-Cai/Open-Jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.238364668034295,
    "B": 2.798015682962046,
    "C": 2.248864352801594
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.05225186990574,
    "p50_s_raw": 0.45112593495287,
    "p95_s_adjusted": 1.839913050853647,
    "p95_s_raw": 0.8449565254268235
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-2B plus rank-8 LoRA and trained scalar decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.27712105156903,
      76.2730193438662
     ],
     "I": [
      16.64935744693625,
      25.12585533438709
     ],
     "capability": [
      42.961468081385505,
      49.91119595082718
     ],
     "composite": {
      "A": [
       1.6444221674835224,
       4.56371492730698
      ],
      "B": [
       1.3617013504482853,
       4.047573981646406
      ],
      "C": [
       1.141959838530224,
       3.1692464772965137
      ]
     },
     "seed": 78
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.61830158128652,
     "capability": 47.71137138509946,
     "intelligence": 21.804441188912403
    },
    "per_type": {
     "choice": {
      "accuracy": 0.496,
      "calibration": 74.56872720500726,
      "cc": 35.545598363062794,
      "cc_by_type": {
       "choice": 35.545598363062794
      },
      "cc_tiered": 38.284840854753284,
      "cc_tiered_open": 41.30120233813892,
      "cc_tiered_sealed": 35.268479371367654,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 35.545598363062794
     },
     "noul": {
      "accuracy": 0.49866666666666665,
      "calibration": 65.4910312846541,
      "cc": -0.2666666666666706,
      "cc_by_type": {
       "noul": -0.2666666666666706
      },
      "cc_tiered": 8.563146997929609,
      "cc_tiered_open": 17.126293995859218,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.456,
      "calibration": 80.79514625419819,
      "cc": 15.96828593404923,
      "cc_by_type": {
       "score": 15.96828593404923
      },
      "cc_tiered": 18.86836867750906,
      "cc_tiered_open": 23.84062554067206,
      "cc_tiered_sealed": 13.89611181434606,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 15.96828593404923
     }
    },
    "raw_on_A": {
     "calibration": 74.44730043893902,
     "capability": 49.372558434366844,
     "intelligence": 24.297816429794665
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.7991313177727157,
    "Emphasis on Accuracy 60:20:20": 2.298491505274465,
    "Emphasis on Cost 20:20:60": 2.7781957464796205,
    "Emphasis on Speed 20:60:20": 3.6134049343333388,
    "Intelligence only": 1.812285794889051
   }
  },
  {
   "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r45/outputs/meta/specs/open-jev-zefan-27b-v1.1.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zefan Cai",
   "axes": {
    "calibration": 83.46836347105969,
    "cost": 14.357426070787312,
    "intelligence": 52.45777412397041,
    "speed": 72.14822609431056
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09909255777071642,
      "ece_score": 80.18148844585671,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.18148844585671,
      "score_before_time_limit": 80.18148844585671,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.32884097035040427,
      "brier": 0.10908145798489985,
      "ece": 0.08491571210766313,
      "n": 371,
      "score": 83.01685757846738,
      "score_before_time_limit": 83.01685757846738,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.991869918699187,
      "interval90_width": 0.5313911472448059,
      "n": 369,
      "nrps": 0.07069056775911403,
      "score": 87.20674438885496,
      "score_before_time_limit": 87.20674438885496,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09258727223189336
     }
    },
    "score": 83.46836347105969
   },
   "capability": 67.96306879751505,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     2.554878535669679,
     2.9114346774435416
    ],
    "B": [
     2.707452757388633,
     3.145833616548021
    ],
    "C": [
     1.7742212053261661,
     2.5838137431054085
    ]
   },
   "cost": {
    "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): Frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price (USD 0.42/M input, 3.00/M output) applied to 2,767,510 input tokens and zero output tokens over 1,624 decisions; self-hosted estimate, not measured hosted billing. Lineage verified by architecture; no pricing gate. The author method encodes each candidate independently.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.7157353448275862
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open-Jev 27B v1.1 (Zefan Cai, LoRA + decision head on Qwen3.8-27B)",
   "endpoint_condition": "Self-hosted author loader, Transformers reference path on RTX PRO 6000 Blackwell; speed is a LOWER BOUND: flash-linear-attention and causal-conv1d kernels were not installed. The author documents the kernel path as answer-identical. Each candidate is encoded independently, producing high input-token usage.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 55.382187378509315,
    "I_sealed": 49.533360869431505,
    "base": 52.45777412397041,
    "excess": 3.275438866639566,
    "gap": 5.84882650907781,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 83.22293323536725,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 88.13447593935398,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 34.03933747412009,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 65.21739130434783,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 48.884291426040605,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.782717441246206,
       "hard": 50.66880107494198,
       "judge": 50.08265733375704,
       "standard": 42.568510259060396
      }
     },
     "sealed|choice": {
      "cc": 75.06513860323663,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 70.9841628959276,
       "judge": 79.51582867783985,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 22.546963950283207,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": 26.21359223300972,
       "judge": 18.421052631578938,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 50.987980054774674,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.75384354090657,
       "hard": 44.757967203121375,
       "judge": 56.3751769445289,
       "standard": 51.48427868038395
      }
     }
    }
   },
   "jevbench_score": 2.8791342364230044,
   "key": "open-jev-zefan-27b-v1.1",
   "last_measured_on": "2026-10-04",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "HF ZefanCai/Open-Jev-27B-v1.1@28cf73067d5b337860bbef3c85b8b82ba8730956; base Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; author loader Zefan-Cai/Open-Jev@3308a15ccd7eea1df7a37d6ddc39b023b801ba16",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9437751004016064,
    "decisive_rate": 0.6711590296495957,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.46836347105969,
     "cost": 14.357426070787312,
     "intelligence": 52.45777412397041,
     "speed": 72.50871441156775
    },
    "board_release": "v1.6.43",
    "capability": 67.96306879751505,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      2.556384162487102,
      2.91320674604079
     ],
     "B": [
      2.708805295775235,
      3.1474885939684447
     ],
     "C": [
      1.7752667795049322,
      2.5853832263765106
     ]
    },
    "jevbench_score": 2.8808671915746493,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 153,
    "rank": 194,
    "ranks": {
     "A": 194,
     "B": 188,
     "C": 194,
     "capability": 92
    },
    "scores": {
     "A": 2.8808671915746493,
     "B": 3.0870580707350324,
     "C": 2.202117328789266
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.7305992921814322,
     "p50_s_raw": 0.7902996460907161,
     "p95_s_adjusted": 3.242888652905821,
     "p95_s_raw": 1.5464443264529106
    },
    "views": {
     "Balanced 33:33:33": 2.4131836334137757,
     "Emphasis on Accuracy 60:20:20": 2.9315893503528665,
     "Emphasis on Cost 20:20:60": 1.7049710123080262,
     "Emphasis on Speed 20:60:20": 3.1691813006586966,
     "Intelligence only": 4.3253676470159474
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 217,
   "ranked": true,
   "ranks": {
    "A": 217,
    "B": 211,
    "C": 216,
    "capability": 107
   },
   "repo": "https://huggingface.co/ZefanCai/Open-Jev-27B-v1.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 2.8791342364230044,
    "B": 3.08546601698711,
    "C": 2.2007926684298393
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.787382136657834,
    "p50_s_raw": 0.818691068328917,
    "p95_s_adjusted": 3.411614702455699,
    "p95_s_raw": 1.6308073512278496
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.8-27B (lineage verified by architecture)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.1715005785796,
      85.37530323313025
     ],
     "I": [
      47.50165176644351,
      56.56652604734154
     ],
     "capability": [
      64.35002446863409,
      69.77751267945519
     ],
     "composite": {
      "A": [
       2.554878535669679,
       2.9114346774435416
      ],
      "B": [
       2.707452757388633,
       3.145833616548021
      ],
      "C": [
       1.7742212053261661,
       2.5838137431054085
      ]
     },
     "seed": 77
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.46836347105969,
     "capability": 67.96306879751505,
     "intelligence": 52.45777412397041
    },
    "per_type": {
     "choice": {
      "accuracy": 0.816,
      "calibration": 80.18148844585671,
      "cc": 76.46902797381657,
      "cc_by_type": {
       "choice": 76.46902797381657
      },
      "cc_tiered": 79.14403591930194,
      "cc_tiered_open": 83.22293323536725,
      "cc_tiered_sealed": 75.06513860323663,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.46902797381657
     },
     "noul": {
      "accuracy": 0.6266666666666667,
      "calibration": 83.01685757846738,
      "cc": 25.333333333333343,
      "cc_by_type": {
       "noul": 25.333333333333343
      },
      "cc_tiered": 28.293150712201648,
      "cc_tiered_open": 34.03933747412009,
      "cc_tiered_sealed": 22.546963950283207,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.333333333333343
     },
     "score": {
      "accuracy": 0.608,
      "calibration": 87.20674438885496,
      "cc": 50.311304174784674,
      "cc_by_type": {
       "score": 50.311304174784674
      },
      "cc_tiered": 49.936135740407636,
      "cc_tiered_open": 48.884291426040605,
      "cc_tiered_sealed": 50.987980054774674,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.311304174784674
     }
    },
    "raw_on_A": {
     "calibration": 81.23345471191158,
     "capability": 67.87352962184931,
     "intelligence": 54.51360453178707
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.411562458800795,
    "Emphasis on Accuracy 60:20:20": 2.9301535776975727,
    "Emphasis on Cost 20:20:60": 1.7044852739371212,
    "Emphasis on Speed 20:60:20": 3.164153037731097,
    "Intelligence only": 4.3253676470159474
   }
  },
  {
   "adapter_id": "images/raw-phi-4-mini/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Microsoft / neutral reproduction",
   "axes": {
    "calibration": 57.015465878675315,
    "cost": 53.22051180195924,
    "intelligence": 13.660237978082595,
    "speed": 91.5252083185614
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.24150351750278642,
      "ece_score": 51.69929649944272,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 51.69929649944272,
      "score_before_time_limit": 51.69929649944272,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.49595687331536387,
      "brier": 0.28688067959784064,
      "ece": 0.2513310447491633,
      "n": 371,
      "score": 49.73379105016734,
      "score_before_time_limit": 49.73379105016734,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8970189701897019,
      "interval90_width": 0.5808491418247513,
      "n": 369,
      "nrps": 0.15574884690331758,
      "score": 69.6133100864159,
      "score_before_time_limit": 69.6133100864159,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.22599247568418224
     }
    },
    "score": 57.015465878675315
   },
   "capability": 35.33785192837895,
   "class": "raw-logit-control",
   "composite_ci95": {
    "A": [
     0.9684805743700438,
     4.125776981199927
    ],
    "B": [
     0.7191616490856636,
     3.3088114802645325
    ],
    "C": [
     0.6725559544236414,
     2.8651229036110593
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03625068965517241
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Phi-4 mini direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 15.681632538277219,
    "I_sealed": 11.638843417887971,
    "base": 13.660237978082595,
    "excess": 1.469401477951004,
    "gap": 4.042789120389248,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 27.48112242812705,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 26.05405405405406,
       "hard": 16.955017301038072,
       "judge": 28.80685563612393,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -56.52173913043479,
       "judge": -50.0,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 19.563775186704607,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.64212383139005,
       "hard": 14.241949100287254,
       "judge": 14.5533398786592,
       "standard": 21.683905999264695
      }
     },
     "sealed|choice": {
      "cc": 25.835499598852735,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 37.47690488507872,
       "hard": 17.533936651583694,
       "judge": 26.443202979515842,
       "standard": 35.70636777928318
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 4.0000000000000036,
       "hard": -66.99029126213591,
       "judge": -34.210526315789465,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 9.081030654811178,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.298756504384265,
       "hard": 1.511859550820871,
       "judge": 5.6438828552519915,
       "standard": 8.766231637344024
      }
     }
    }
   },
   "jevbench_score": 2.4785276781687653,
   "key": "raw-phi-4-mini",
   "last_measured_on": "2026-10-02",
   "licence": "MIT",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6096256684491979,
    "decisive_rate": 0.5040431266846361,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 57.015465878675315,
     "cost": 53.22051180195924,
     "intelligence": 13.660237978082595,
     "speed": 91.89255362835293
    },
    "board_release": "v1.6.43",
    "capability": 35.33785192837895,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 122,
    "capability_rank": 150,
    "composite_ci95": {
     "A": [
      0.9687582607873362,
      4.12745446682704
     ],
     "B": [
      0.7192841291179243,
      3.3096762413612253
     ],
     "C": [
      0.672748792213428,
      2.866287824185444
     ]
    },
    "jevbench_score": 2.4794266829120515,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 154,
    "rank": 195,
    "ranks": {
     "A": 195,
     "B": 195,
     "C": 195,
     "capability": 216
    },
    "scores": {
     "A": 2.4794266829120515,
     "B": 1.927494035430626,
     "C": 1.721824085355591
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23357354765757918,
     "p50_s_raw": 0.04178677382878959,
     "p95_s_adjusted": 0.27689874032512307,
     "p95_s_raw": 0.06344937016256152
    },
    "views": {
     "Balanced 33:33:33": 2.1766003082696974,
     "Emphasis on Accuracy 60:20:20": 1.497083363151444,
     "Emphasis on Cost 20:20:60": 2.657076827352907,
     "Emphasis on Speed 20:60:20": 2.994215323690922,
     "Intelligence only": 1.0196116461240732
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 218,
   "ranked": true,
   "ranks": {
    "A": 218,
    "B": 218,
    "C": 218,
    "capability": 239
   },
   "repo": "https://huggingface.co/microsoft/Phi-4-mini-instruct",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 2.4785276781687653,
    "B": 1.9270593300106893,
    "C": 1.7211997765060865
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23720237296074628,
    "p50_s_raw": 0.043601186480373144,
    "p95_s_adjusted": 0.2967290996108204,
    "p95_s_raw": 0.0733645498054102
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Phi-4-mini-instruct BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      51.56094132127967,
      61.43189419566834
     ],
     "I": [
      9.603807805278013,
      16.664196620072826
     ],
     "capability": [
      31.514955827472072,
      38.142435957298304
     ],
     "composite": {
      "A": [
       0.9684805743700438,
       4.125776981199927
      ],
      "B": [
       0.7191616490856636,
       3.3088114802645325
      ],
      "C": [
       0.6725559544236414,
       2.8651229036110593
      ]
     },
     "seed": 93
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 57.015465878675315,
     "capability": 35.33785192837895,
     "intelligence": 13.660237978082595
    },
    "per_type": {
     "choice": {
      "accuracy": 0.42133333333333334,
      "calibration": 51.69929649944272,
      "cc": 25.996798120553574,
      "cc_by_type": {
       "choice": 25.996798120553574
      },
      "cc_tiered": 26.658311013489893,
      "cc_tiered_open": 27.48112242812705,
      "cc_tiered_sealed": 25.835499598852735,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 25.996798120553574
     },
     "noul": {
      "accuracy": 0.304,
      "calibration": 49.73379105016734,
      "cc": -39.2,
      "cc_by_type": {
       "noul": -39.2
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.4026666666666667,
      "calibration": 69.6133100864159,
      "cc": 13.214800471337696,
      "cc_by_type": {
       "score": 13.214800471337696
      },
      "cc_tiered": 14.322402920757892,
      "cc_tiered_open": 19.563775186704607,
      "cc_tiered_sealed": 9.081030654811178,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 13.214800471337696
     }
    },
    "raw_on_A": {
     "calibration": 53.16092441580774,
     "capability": 33.04410281531054,
     "intelligence": 12.927281214813327
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.1756766130294327,
    "Emphasis on Accuracy 60:20:20": 1.496821108735571,
    "Emphasis on Cost 20:20:60": 2.6562508272129657,
    "Emphasis on Speed 20:60:20": 2.9910709227275794,
    "Intelligence only": 1.0196116461240732
   }
  },
  {
   "adapter_id": "images/simplejev-qwen3.5-0.8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "sabeel111 / Featherless AI",
   "axes": {
    "calibration": 49.46821562395294,
    "cost": 70.31175519898045,
    "intelligence": 12.67752479561282,
    "speed": 58.64501925086723
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22436826460136267,
      "ece_score": 55.126347079727466,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 54.832339895302255,
      "score_before_time_limit": 55.126347079727466,
      "time_limit_failed_share": 0.005333333333333333
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.40897819205718194,
      "ece": 0.4071351151116132,
      "n": 371,
      "score": 18.523449039070215,
      "score_before_time_limit": 18.572976977677357,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.8807588075880759,
      "interval90_width": 0.6312104787714541,
      "n": 369,
      "nrps": 0.17353025104683517,
      "score": 75.04885793748636,
      "score_before_time_limit": 76.89432165726062,
      "time_limit_failed_share": 0.024,
      "top_ece": 0.14429165790397622
     }
    },
    "score": 49.46821562395294
   },
   "capability": 31.07287020978288,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.6401075645706304,
     4.2421555906986015
    ],
    "B": [
     0.4723465255880033,
     3.4384124482980876
    ],
    "C": [
     0.44451914206293786,
     2.945941382429584
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009763559113300492
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 14,
    "n_over_limit_answered_ok": 14,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SimpleJev (Qwen3.5-0.8B, CPU)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 16.855956156782675,
    "I_sealed": 8.499093434442965,
    "base": 12.67752479561282,
    "excess": 5.783475079901466,
    "gap": 8.35686272233971,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 16.101545239125173,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.702702702702708,
       "hard": 13.180245360176167,
       "judge": 28.80685563612393,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 31.332643202208423,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": 30.000000000000004,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 3.1336800290144353,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 21.659111114444617,
       "hard": -15.769090377553162,
       "judge": 24.5874654817379,
       "standard": -0.504172879650655
      }
     },
     "sealed|choice": {
      "cc": 16.18809910474239,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 10.87133249575052,
       "hard": 9.389140271493197,
       "judge": 19.92551210428307,
       "standard": 26.838280576425678
      }
     },
     "sealed|noul": {
      "cc": 9.309181198586508,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 24.0,
       "hard": 12.621359223300965,
       "judge": 5.263157894736836,
       "standard": 1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 19.829285077433635,
       "hard": -17.18882511862938,
       "judge": 2.232240541966324,
       "standard": -0.6083842924782035
      }
     }
    }
   },
   "jevbench_score": 1.972490934796093,
   "key": "simplejev-qwen3.5-0.8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 server; Apache-2.0 Qwen3.5-0.8B checkpoint",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.581081081081081,
    "decisive_rate": 1.0,
    "supported": true,
    "valid": 370
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 50.197881904888476,
     "cost": 70.31175519898045,
     "intelligence": 13.000426205687345,
     "speed": 58.929227255635794
    },
    "board_release": "v1.6.43",
    "capability": 31.59915405528791,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.709004207843539,
      4.446455346949053
     ],
     "B": [
      0.5274465681104142,
      3.6235557003922736
     ],
     "C": [
      0.49236403322467986,
      3.0878162131590647
     ]
    },
    "jevbench_score": 2.112084174105171,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 155,
    "rank": 197,
    "ranks": {
     "A": 197,
     "B": 197,
     "C": 197,
     "capability": 225
    },
    "scores": {
     "A": 2.112084174105171,
     "B": 1.649257467523907,
     "C": 1.4667251209063683
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 8.797023160476238,
     "p50_s_raw": 4.323511580238119,
     "p95_s_adjusted": 14.545931534096598,
     "p95_s_raw": 7.197965767048299
    },
    "views": {
     "Balanced 33:33:33": 1.8759479039856477,
     "Emphasis on Accuracy 60:20:20": 1.2903897195924547,
     "Emphasis on Cost 20:20:60": 2.475315897383538,
     "Emphasis on Speed 20:60:20": 2.3795755523953557,
     "Intelligence only": 0.8788864373471849
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 220,
   "ranked": true,
   "ranks": {
    "A": 220,
    "B": 220,
    "C": 220,
    "capability": 248
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 1.972490934796093,
    "B": 1.5361595548959908,
    "C": 1.3697853713861756
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 9.16144129736349,
    "p50_s_raw": 4.505720648681745,
    "p95_s_adjusted": 14.911944263335316,
    "p95_s_raw": 7.380972131667658
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-0.8B through SimpleJev native assistant-prefill option-logit scorer",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      43.866426363812614,
      53.741885017483725
     ],
     "I": [
      8.322750852470811,
      16.973486697243555
     ],
     "capability": [
      26.61188293230259,
      34.97208009895024
     ],
     "composite": {
      "A": [
       0.6401075645706304,
       4.2421555906986015
      ],
      "B": [
       0.4723465255880033,
       3.4384124482980876
      ],
      "C": [
       0.44451914206293786,
       2.945941382429584
      ]
     },
     "seed": 100
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 49.46821562395294,
     "capability": 31.07287020978288,
     "intelligence": 12.67752479561282
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 54.832339895302255,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 16.144822171933782,
      "cc_tiered_open": 16.101545239125173,
      "cc_tiered_sealed": 16.18809910474239,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.5733333333333334,
      "calibration": 18.523449039070215,
      "cc": 14.666666666666671,
      "cc_by_type": {
       "noul": 14.666666666666671
      },
      "cc_tiered": 20.320912200397466,
      "cc_tiered_open": 31.332643202208423,
      "cc_tiered_sealed": 9.309181198586508,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 14.666666666666671
     },
     "score": {
      "accuracy": 0.3253333333333333,
      "calibration": 75.04885793748636,
      "cc": -3.0837213034381916,
      "cc_by_type": {
       "score": -3.0837213034381916
      },
      "cc_tiered": 1.5668400145072177,
      "cc_tiered_open": 3.1336800290144353,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 51.07750814673811,
     "capability": 31.902855962425715,
     "intelligence": 12.728203778113318
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.022666666666666613,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.040000000000000036,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 1.7508559727605375,
    "Emphasis on Accuracy 60:20:20": 1.1997880764128812,
    "Emphasis on Cost 20:20:60": 2.31920980732424,
    "Emphasis on Speed 20:60:20": 2.228234749472427,
    "Intelligence only": 0.8150108628546057
   }
  },
  {
   "adapter_id": "images/smalljev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Aditya (isHeSatoshi)",
   "axes": {
    "calibration": 77.23092648324078,
    "cost": 60.77469204714053,
    "intelligence": 8.670533657732655,
    "speed": 90.11972690216095
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1267499372828752,
      "ece_score": 74.65001254342496,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 74.65001254342496,
      "score_before_time_limit": 74.65001254342496,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.7653333333333333,
      "brier": 0.252171281098217,
      "ece": 0.15089599377413593,
      "n": 375,
      "score": 69.82080124517282,
      "score_before_time_limit": 69.82080124517282,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.8366666666666663,
      "n": 375,
      "nrps": 0.15538624289679162,
      "score": 87.22196566112457,
      "score_before_time_limit": 87.22196566112457,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05008722194035847
     }
    },
    "score": 77.23092648324078
   },
   "capability": 42.95073007048671,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.24656367704805998,
     1.643207687945247
    ],
    "B": [
     0.1702968609786506,
     1.2179490503049588
    ],
    "C": [
     0.17122477572781938,
     1.1411164499619768
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020300665024630543
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "smalljev semantic-v9",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 10.044883698522286,
    "I_sealed": 7.296183616943025,
    "base": 8.670533657732655,
    "excess": 0.17531243914101724,
    "gap": 2.748700081579261,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 18.8258695634495,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 13.180245360176167,
       "judge": 28.80685563612393,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -73.91304347826086,
       "judge": -80.0,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 11.308781532117358,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 1.465713661728496,
       "hard": 15.552501797814056,
       "judge": 21.821381656800277,
       "standard": -8.026025251105985
      }
     },
     "sealed|choice": {
      "cc": 12.092078018745205,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 37.47690488507872,
       "hard": 8.37104072398188,
       "judge": 6.890130353817524,
       "standard": 14.644660672496622
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -60.0,
       "hard": -82.52427184466019,
       "judge": -52.63157894736843,
       "standard": -52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 9.796472832083872,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 12.348066695721426,
       "hard": 6.731683495180563,
       "judge": 16.491346453277444,
       "standard": 4.607944142281362
      }
     }
    }
   },
   "jevbench_score": 0.7718897362808083,
   "key": "smalljev",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7272727272727273,
    "decisive_rate": 0.23466666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.23092648324078,
     "cost": 60.77469204714053,
     "intelligence": 8.670533657732655,
     "speed": 90.21131723708834
    },
    "board_release": "v1.6.43",
    "capability": 42.95073007048671,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 107,
    "capability_rank": 135,
    "composite_ci95": {
     "A": [
      0.2465766163795243,
      1.6433530428400023
     ],
     "B": [
      0.17030179892922623,
      1.2180129321235638
     ],
     "C": [
      0.17123376137466964,
      1.1412173908611125
     ]
    },
    "jevbench_score": 0.7719455446134506,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 157,
    "rank": 200,
    "ranks": {
     "A": 200,
     "B": 200,
     "C": 200,
     "capability": 198
    },
    "scores": {
     "A": 0.7719455446134506,
     "B": 0.5545056442894855,
     "C": 0.5360732948704519
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2831438841298223,
     "p50_s_raw": 0.06657194206491113,
     "p95_s_adjusted": 0.33640395337715745,
     "p95_s_raw": 0.09320197668857871
    },
    "views": {
     "Balanced 33:33:33": 0.6314288092197278,
     "Emphasis on Accuracy 60:20:20": 0.4025186185800094,
     "Emphasis on Cost 20:20:60": 0.8553620766195851,
     "Emphasis on Speed 20:60:20": 0.9110150146047675,
     "Intelligence only": 0.2607338855207101
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 222,
   "ranked": true,
   "ranks": {
    "A": 222,
    "B": 222,
    "C": 222,
    "capability": 220
   },
   "repo": "https://github.com/isHeSatoshi/smalljev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.7718897362808083,
    "B": 0.5544826064895818,
    "C": 0.5360345390838948
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28393019186332824,
    "p50_s_raw": 0.06696509593166411,
    "p95_s_adjusted": 0.34262238498777153,
    "p95_s_raw": 0.09631119249388576
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "MiniCPM5-2B-Base, 2.5B dense, with LoRA and native semantic decision heads",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.4661928849833,
      79.90983904292283
     ],
     "I": [
      5.751868300889253,
      11.443348446757808
     ],
     "capability": [
      39.42005219873977,
      44.84742177133798
     ],
     "composite": {
      "A": [
       0.24656367704805998,
       1.643207687945247
      ],
      "B": [
       0.1702968609786506,
       1.2179490503049588
      ],
      "C": [
       0.17122477572781938,
       1.1411164499619768
      ]
     },
     "seed": 101
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.23092648324078,
     "capability": 42.95073007048671,
     "intelligence": 8.670533657732655
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 74.65001254342496,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 15.458973791097353,
      "cc_tiered_open": 18.8258695634495,
      "cc_tiered_sealed": 12.092078018745205,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.17066666666666666,
      "calibration": 69.82080124517282,
      "cc": -65.86666666666667,
      "cc_by_type": {
       "noul": -65.86666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.3626666666666667,
      "calibration": 87.22196566112457,
      "cc": 9.603703249850426,
      "cc_by_type": {
       "score": 9.603703249850426
      },
      "cc_tiered": 10.552627182100615,
      "cc_tiered_open": 11.308781532117358,
      "cc_tiered_sealed": 9.796472832083872,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 9.603703249850426
     }
    },
    "raw_on_A": {
     "calibration": 71.91610164610462,
     "capability": 40.0721638349662,
     "intelligence": 8.22822602382777
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.6313790228575082,
    "Emphasis on Accuracy 60:20:20": 0.40250647893597635,
    "Emphasis on Cost 20:20:60": 0.8553072591161187,
    "Emphasis on Speed 20:60:20": 0.9108284924171282,
    "Intelligence only": 0.2607338855207101
   }
  },
  {
   "adapter_id": "images/kev-0.6b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 66.26655558235618,
    "cost": 80.09400261254245,
    "intelligence": 7.598789169415934,
    "speed": 91.48734752715623
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20778793892461145,
      "ece_score": 58.44241221507771,
      "mean_tvd": null,
      "n": 731,
      "n_ece_hard": 306,
      "n_tvd": 0,
      "score": 58.44241221507771,
      "score_before_time_limit": 58.44241221507771,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.568733153638814,
      "brier": 0.2628086253369272,
      "ece": 0.18765498652291104,
      "n": 371,
      "score": 62.4690026954178,
      "score_before_time_limit": 62.4690026954178,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9320652173913043,
      "interval90_width": 0.8045742753623186,
      "n": 368,
      "nrps": 0.17410404691603434,
      "score": 77.88825183657305,
      "score_before_time_limit": 77.88825183657305,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1340654581762524
     }
    },
    "score": 66.26655558235618
   },
   "capability": 36.93267237588606,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.13850142803501514,
     1.459069355435144
    ],
    "B": [
     0.09387250382633613,
     1.0709271119347388
    ],
    "C": [
     0.09618154724653831,
     1.0132426079410723
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004608220443349753
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 0.6B (research preview)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 8.945068603011505,
    "I_sealed": 6.252509735820362,
    "base": 7.598789169415934,
    "excess": 0.1191712247528991,
    "gap": 2.692558867191143,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 16.179258482292497,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 45.513513513513516,
       "hard": 9.405473419314262,
       "judge": 24.851680949241928,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0.0,
       "hard": -73.91304347826086,
       "judge": -60.0,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 10.655947326742021,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 23.83763830928548,
       "hard": -1.1034131190897023,
       "judge": 28.04844552251834,
       "standard": 1.4950754334692617
      }
     },
     "sealed|choice": {
      "cc": 18.757529207461086,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 34.8163476461459,
       "hard": 6.334841628959255,
       "judge": 24.581005586592198,
       "standard": 26.838280576425678
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -4.0000000000000036,
       "hard": -66.99029126213591,
       "judge": -44.73684210526315,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 11.09665079983172,
       "hard": -14.21346816054152,
       "judge": 12.732793638645301,
       "standard": -1.8098672811444283
      }
     }
    }
   },
   "jevbench_score": 0.543110767402779,
   "key": "kev-0.6b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.69375,
    "decisive_rate": 0.431266846361186,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 66.26655558235618,
     "cost": 80.09400261254245,
     "intelligence": 7.598789169415934,
     "speed": 91.51497362122684
    },
    "board_release": "v1.6.43",
    "capability": 36.93267237588606,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 120,
    "capability_rank": 148,
    "composite_ci95": {
     "A": [
      0.13850323393885078,
      1.4591062434238613
     ],
     "B": [
      0.09387316749385785,
      1.070943045752662
     ],
     "C": [
      0.09618280134642418,
      1.0132682245999038
     ]
    },
    "jevbench_score": 0.5431213026023877,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 160,
    "rank": 203,
    "ranks": {
     "A": 203,
     "B": 203,
     "C": 203,
     "capability": 212
    },
    "scores": {
     "A": 0.5431213026023877,
     "B": 0.382771242886796,
     "C": 0.377167571251658
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26348884413018825,
     "p50_s_raw": 0.05674442206509411,
     "p95_s_adjusted": 0.2677567361854017,
     "p95_s_raw": 0.05887836809270084
    },
    "views": {
     "Balanced 33:33:33": 0.4469959095569562,
     "Emphasis on Accuracy 60:20:20": 0.2761353719898703,
     "Emphasis on Cost 20:20:60": 0.6416336282227073,
     "Emphasis on Speed 20:60:20": 0.6529389194958221,
     "Intelligence only": 0.17550648828077675
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 226,
   "ranked": true,
   "ranks": {
    "A": 226,
    "B": 226,
    "C": 226,
    "capability": 234
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.543110767402779,
    "B": 0.3827670566734275,
    "C": 0.37716025514081863
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26317979032173755,
    "p50_s_raw": 0.056589895160868764,
    "p95_s_adjusted": 0.2697818389628083,
    "p95_s_raw": 0.059890919481404126
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-0.6B-Base + LoRA + learned pointer head; jaredpalmer/kev-0.6b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      60.54900964119034,
      70.35051467735538
     ],
     "I": [
      4.680395319947538,
      10.897085922837281
     ],
     "capability": [
      33.22958246000976,
      39.72314225130261
     ],
     "composite": {
      "A": [
       0.13850142803501514,
       1.459069355435144
      ],
      "B": [
       0.09387250382633613,
       1.0709271119347388
      ],
      "C": [
       0.09618154724653831,
       1.0132426079410723
      ]
     },
     "seed": 53
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.26655558235618,
     "capability": 36.93267237588606,
     "intelligence": 7.598789169415934
    },
    "per_type": {
     "choice": {
      "accuracy": 0.368,
      "calibration": 58.44241221507771,
      "cc": 19.176226518761275,
      "cc_by_type": {
       "choice": 19.176226518761275
      },
      "cc_tiered": 17.468393844876793,
      "cc_tiered_open": 16.179258482292497,
      "cc_tiered_sealed": 18.757529207461086,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.176226518761275
     },
     "noul": {
      "accuracy": 0.296,
      "calibration": 62.4690026954178,
      "cc": -40.800000000000004,
      "cc_by_type": {
       "noul": -40.800000000000004
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.30933333333333335,
      "calibration": 77.88825183657305,
      "cc": -0.08941736278893497,
      "cc_by_type": {
       "score": -0.08941736278893497
      },
      "cc_tiered": 5.327973663371011,
      "cc_tiered_open": 10.655947326742021,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 65.17717796738081,
     "capability": 37.13879390532707,
     "intelligence": 9.100409843273326
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.02533333333333332,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.44698639487866926,
    "Emphasis on Accuracy 60:20:20": 0.27613319333637887,
    "Emphasis on Cost 20:20:60": 0.6416218653396832,
    "Emphasis on Speed 20:60:20": 0.6529023777269423,
    "Intelligence only": 0.17550648828077675
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Julien Lestel / @jlestel (JuL)",
   "axes": {
    "calibration": 77.79406843975872,
    "cost": 68.9870979950853,
    "intelligence": 7.4645122320968404,
    "speed": 90.5592978634667
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.049032187499999956,
      "ece_score": 90.19356250000001,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 90.19356250000001,
      "score_before_time_limit": 90.19356250000001,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.7013333333333334,
      "brier": 0.20025538554666658,
      "ece": 0.08354026666666664,
      "n": 375,
      "score": 83.29194666666668,
      "score_before_time_limit": 83.29194666666668,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8418230563002681,
      "interval90_width": 0.7091152815013402,
      "n": 373,
      "nrps": 0.22054212385388733,
      "score": 59.89669615260948,
      "score_before_time_limit": 60.70611096548258,
      "time_limit_failed_share": 0.013333333333333334,
      "top_ece": 0.2826678284182305
     }
    },
    "score": 77.79406843975872
   },
   "capability": 42.62929033592778,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.23984046541310475,
     1.1184890962978362
    ],
    "B": [
     0.16476538317869,
     0.8100368206640075
    ],
    "C": [
     0.16655587875910055,
     0.7767285390957197
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no bookable listing for this base). openbmb/MiniCPM5-2B is an EXPLICIT entry in the frozen module's UNLISTED_BASE_MODELS table, so pricing_v15.reference() returns None without raising: the decision that there is no bookable listing for this base is already taken, and the only MiniCPM key in either dated snapshot is deepinfra:openbmb/MiniCPM-Llama3-V-2_5 at (0.34, 0.34), a VISION model and not a comparable reference. The row therefore carries its own labelled size-class estimate: deepinfra:Qwen/Qwen3.5-2B at USD 0.02 per 1M input and USD 0.10 per 1M output in the frozen 25 Sep snapshot - a 2B dense instruct model, the same size class as this 2B base - x this row's own measured tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw. RELEASE-LANE LINE, recorded and not acted on: if a bookable MiniCPM5-2B text listing has appeared since 25 Sep, this row reprices by one multiplication from the raw. receipts/COST-ALTERNATIVES-R58.json has the arithmetic for the alternatives already done.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.016866253333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010808449559918753
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 5,
    "n_over_limit_answered_ok": 4,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jul fast (usejul/jul-decision-minicpm5-2b: MiniCPM5-2B fine-tune with a pointer head, option probabilities from one pass, self-hosted via `jul serve`)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,498 of 1,500 answered. Library github.com/usejul/jul tag v0.5.1, PyPI jul==0.5.1, Apache-2.0; weights usejul/jul-decision-minicpm5-2b at f556a6936089693ea68bfed3080c637e4853a3ff, Apache-2.0; base openbmb/MiniCPM5-2B at 12a3808a. NO MAPPING DECISION OF OURS: `jul serve` speaks the TypeSafe /v1/systemone wire format, so the UNCHANGED jevbench.adapters.typesafe applies, and all eight checks of the unamended smoke gate passed on synthetic requests before any sealed item was sent. STACK: the author's own published install line from their request, `pip install \"jul[torch]==0.5.1\"` in an isolated venv (torch 2.14.1+cu130, transformers 5.19.0), served with their own published command verbatim (`jul serve --model jul-decision-minicpm5-2b --host 127.0.0.1 --port 8577`). One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW IS THE FIRST THAT NEITHER REFUSES NOR UNIFORMLY TRUNCATES THEM, AND THE SPLIT IS BY QUESTION TYPE. The author declares a 2,048-token state budget with no refusal - 'the beginning of the text is kept and the end is cut, without an error'. Their server's own usage.input_tokens shows that budget applied to 17 of the 23 (counts 2,096-2,135) and NOT to the other 6, which were read at 107,331-107,997 tokens; those 6 are exactly the 6 score questions among the 23, while all 17 truncated ones are choice or noul. Two of the six full reads then failed with the server's own CUDA OutOfMemoryError on a 32 GB card ('Tried to allocate 5.59 GiB' / '5.41 GiB'), which is this row's only 2 invalid and which a larger card would most likely have answered: the cause is the score path not applying the author's own documented truncation, and the author's note that '24 GB is enough' holds only under that truncation. The 2 invalid are counted wrong, the conservative direction, and are 0.13 % of the pool. PROBABILITIES ARE RETURNED ROUNDED TO FOUR DECIMALS: 426 of 1,498 answered distributions sum to 1 +/- 2e-4 (observed range 0.9998-1.0002); none is off by more than that, so it is the server's rounding and not a partial readout. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 58, GitHub fstandhartinger/jevbench#198",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 7.349597785148552,
    "I_sealed": 7.579426679045129,
    "base": 7.4645122320968404,
    "excess": -2.8032165363348214,
    "gap": -0.22982889389657757,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 22.048793355445657,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 15.06763133060712,
       "judge": 16.941331575477918,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -100.0,
       "judge": -50.0,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 26.921515151515163,
       "hard": -16.06981481481484,
       "judge": 27.538983050847456,
       "standard": -57.756296296296284
      }
     },
     "sealed|choice": {
      "cc": 22.73828003713539,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 54.77052693814206,
       "hard": 14.479638009049756,
       "judge": 25.51210428305402,
       "standard": 19.078704273925375
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -4.0000000000000036,
       "hard": -74.75728155339806,
       "judge": -42.10526315789473,
       "standard": -32.3943661971831
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 2.8917730496454075,
       "hard": -23.2233823529411,
       "judge": -11.343473053892229,
       "standard": -27.880304568527947
      }
     }
    }
   },
   "jevbench_score": 0.5172342787655929,
   "key": "jul-fast",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "usejul/jul-decision-minicpm5-2b@f556a6936089693ea68bfed3080c637e4853a3ff",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8928571428571429,
    "decisive_rate": 0.2986666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.06387337738309,
     "cost": 68.9870979950853,
     "intelligence": 7.4645122320968404,
     "speed": 90.69403874154062
    },
    "board_release": "v1.6.43",
    "capability": 42.76419280473996,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 108,
    "capability_rank": 136,
    "composite_ci95": {
     "A": [
      0.2399155159563835,
      1.1190632145905357
     ],
     "B": [
      0.16479417918823877,
      0.8102776687445344
     ],
     "C": [
      0.166607997191933,
      0.7771272323545388
     ]
    },
    "jevbench_score": 0.5174168972906218,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 161,
    "rank": 204,
    "ranks": {
     "A": 204,
     "B": 205,
     "C": 204,
     "capability": 199
    },
    "scores": {
     "A": 0.5174168972906218,
     "B": 0.3638597737908031,
     "C": 0.35931728978515404
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23712351802387274,
     "p50_s_raw": 0.043561759011936374,
     "p95_s_adjusted": 0.3594359368114965,
     "p95_s_raw": 0.10471796840574825
    },
    "views": {
     "Balanced 33:33:33": 0.41923158593664034,
     "Emphasis on Accuracy 60:20:20": 0.26072023907855174,
     "Emphasis on Cost 20:20:60": 0.5912460853155722,
     "Emphasis on Speed 20:60:20": 0.613844381562066,
     "Intelligence only": 0.16636589222451567
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 227,
   "ranked": true,
   "ranks": {
    "A": 227,
    "B": 228,
    "C": 227,
    "capability": 222
   },
   "repo": "https://huggingface.co/usejul/jul-decision-minicpm5-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.5172342787655929,
    "B": 0.363787515323243,
    "C": 0.35919047136499505
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2375775480235461,
    "p50_s_raw": 0.04378877401177306,
    "p95_s_adjusted": 0.3700537570839515,
    "p95_s_raw": 0.11002687854197575
   },
   "status": {
    "answered_ok": 1498,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "openbmb/MiniCPM5-2B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.94462322454935,
      80.16649632842498
     ],
     "I": [
      5.676467601887915,
      9.888156546142561
     ],
     "capability": [
      39.70355516328919,
      44.189161649231124
     ],
     "composite": {
      "A": [
       0.23984046541310475,
       1.1184890962978362
      ],
      "B": [
       0.16476538317869,
       0.8100368206640075
      ],
      "C": [
       0.16655587875910055,
       0.7767285390957197
      ]
     },
     "seed": 74
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.79406843975872,
     "capability": 42.62929033592778,
     "intelligence": 7.4645122320968404
    },
    "per_type": {
     "choice": {
      "accuracy": 0.404,
      "calibration": 90.19356250000001,
      "cc": 23.78011234997108,
      "cc_by_type": {
       "choice": 23.78011234997108
      },
      "cc_tiered": 22.393536696290525,
      "cc_tiered_open": 22.048793355445657,
      "cc_tiered_sealed": 22.73828003713539,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 23.78011234997108
     },
     "noul": {
      "accuracy": 0.26666666666666666,
      "calibration": 83.29194666666668,
      "cc": -46.666666666666664,
      "cc_by_type": {
       "noul": -46.666666666666664
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2826666666666667,
      "calibration": 59.89669615260948,
      "cc": -15.119236842105233,
      "cc_by_type": {
       "score": -15.119236842105233
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 78.08896065787587,
     "capability": 43.460037896212135,
     "intelligence": 8.831115134548394
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.4191884671241191,
    "Emphasis on Accuracy 60:20:20": 0.2607102324554309,
    "Emphasis on Cost 20:20:60": 0.5911946272474554,
    "Emphasis on Speed 20:60:20": 0.613678011636167,
    "Intelligence only": 0.16636589222451567
   }
  },
  {
   "adapter_id": "images/opendecision/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Deepan Wadhwa",
   "axes": {
    "calibration": 68.56606448002526,
    "cost": 79.10704862374335,
    "intelligence": 7.425225288997898,
    "speed": 86.36099014084385
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.18936051642522217,
      "ece_score": 62.12789671495557,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 62.12789671495557,
      "score_before_time_limit": 62.12789671495557,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4933333333333333,
      "brier": 0.2702030712841029,
      "ece": 0.17767917616168663,
      "n": 375,
      "score": 64.46416476766268,
      "score_before_time_limit": 64.46416476766268,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9706666666666667,
      "interval90_width": 0.8675555555555556,
      "n": 375,
      "nrps": 0.1718997562232394,
      "score": 79.10613195745756,
      "score_before_time_limit": 79.10613195745756,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.12298880231380463
     }
    },
    "score": 68.56606448002526
   },
   "capability": 37.99564488451158,
   "class": "classifier",
   "composite_ci95": {
    "A": [
     0.20915427982595058,
     1.1140594117388836
    ],
    "B": [
     0.14360793060164476,
     0.8080395435414361
    ],
    "C": [
     0.1452460276569101,
     0.773652369263114
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004970862068965518
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "OpenDecision (ModernBERT-large zero-shot)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 8.199956224645156,
    "I_sealed": 6.6504943533506395,
    "base": 7.425225288997898,
    "excess": -1.0239257711437273,
    "gap": 1.5494618712945165,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 21.530744131863024,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 37.72972972972973,
       "hard": -3.8062283737024116,
       "judge": 32.76203032300594,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -82.6086956521739,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 3.069124542072446,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 29.089033648823246,
       "hard": -4.629529940171362,
       "judge": 10.925488522812966,
       "standard": -6.32806701792612
      }
     },
     "sealed|choice": {
      "cc": 18.786220529284186,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 34.8163476461459,
       "hard": 9.898190045248851,
       "judge": 19.92551210428307,
       "standard": 26.838280576425678
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 12.00000000000001,
       "hard": -39.80582524271845,
       "judge": -50.0,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 1.165262530767735,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 23.523223159689433,
       "hard": -5.374879041668379,
       "judge": 2.0574839819155533,
       "standard": 1.728233184457384
      }
     }
    }
   },
   "jevbench_score": 0.5084959848229078,
   "key": "opendecision",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6842105263157895,
    "decisive_rate": 0.5066666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 68.56606448002526,
     "cost": 79.10704862374335,
     "intelligence": 7.425225288997898,
     "speed": 87.12856375997052
    },
    "board_release": "v1.6.43",
    "capability": 37.99564488451158,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 117,
    "capability_rank": 145,
    "composite_ci95": {
     "A": [
      0.20924922812147978,
      1.114873836219044
     ],
     "B": [
      0.14364373295734356,
      0.808381046176735
     ],
     "C": [
      0.14531196397324977,
      0.7742179418187807
     ]
    },
    "jevbench_score": 0.508795165212109,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 162,
    "rank": 205,
    "ranks": {
     "A": 205,
     "B": 206,
     "C": 205,
     "capability": 209
    },
    "scores": {
     "A": 0.508795165212109,
     "B": 0.3579489209247008,
     "C": 0.3533299758417423
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29369962783530357,
     "p50_s_raw": 0.07184981391765177,
     "p95_s_adjusted": 0.6595386148197576,
     "p95_s_raw": 0.2547693074098788
    },
    "views": {
     "Balanced 33:33:33": 0.41664409033845246,
     "Emphasis on Accuracy 60:20:20": 0.25754719918644864,
     "Emphasis on Cost 20:20:60": 0.5990327904113044,
     "Emphasis on Speed 20:60:20": 0.6067044473606604,
     "Intelligence only": 0.1637528610901445
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 228,
   "ranked": true,
   "ranks": {
    "A": 228,
    "B": 229,
    "C": 228,
    "capability": 232
   },
   "repo": "https://github.com/deepanwadhwa/OpenDecision",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.5084959848229078,
    "B": 0.3578304284057583,
    "C": 0.3531222116825748
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.335404502786696,
    "p50_s_raw": 0.09270225139334798,
    "p95_s_adjusted": 0.6891790978377684,
    "p95_s_raw": 0.2695895489188842
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "MoritzLaurer/ModernBERT-large-zeroshot-v2.0 (~400M) used as a zero-shot NLI decision engine",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.90521170003539,
      71.97774479185684
     ],
     "I": [
      5.421041180341186,
      9.877186440823028
     ],
     "capability": [
      34.738025189088674,
      40.24525966167053
     ],
     "composite": {
      "A": [
       0.20915427982595058,
       1.1140594117388836
      ],
      "B": [
       0.14360793060164476,
       0.8080395435414361
      ],
      "C": [
       0.1452460276569101,
       0.773652369263114
      ]
     },
     "seed": 80
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 68.56606448002526,
     "capability": 37.99564488451158,
     "intelligence": 7.425225288997898
    },
    "per_type": {
     "choice": {
      "accuracy": 0.36933333333333335,
      "calibration": 62.12789671495557,
      "cc": 19.346740808806086,
      "cc_by_type": {
       "choice": 19.346740808806086
      },
      "cc_tiered": 20.158482330573605,
      "cc_tiered_open": 21.530744131863024,
      "cc_tiered_sealed": 18.786220529284186,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.346740808806086
     },
     "noul": {
      "accuracy": 0.3466666666666667,
      "calibration": 64.46416476766268,
      "cc": -30.666666666666664,
      "cc_by_type": {
       "noul": -30.666666666666664
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.3253333333333333,
      "calibration": 79.10613195745756,
      "cc": 2.4048951483054193,
      "cc_by_type": {
       "score": 2.4048951483054193
      },
      "cc_tiered": 2.1171935364200904,
      "cc_tiered_open": 3.069124542072446,
      "cc_tiered_sealed": 1.165262530767735,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 2.4048951483054193
     }
    },
    "raw_on_A": {
     "calibration": 66.1740117306899,
     "capability": 37.492707526259565,
     "intelligence": 8.811403321829234
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.41637660935819926,
    "Emphasis on Accuracy 60:20:20": 0.2574858508477883,
    "Emphasis on Cost 20:20:60": 0.5987010081282494,
    "Emphasis on Speed 20:60:20": 0.605684594386601,
    "Intelligence only": 0.1637528610901445
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 22.615652963639242,
    "cost": 70.20313863040258,
    "intelligence": 7.802105373202749,
    "speed": 93.07921198047904
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.5367426710097716,
      "ece_score": 0.0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0.0,
      "score_before_time_limit": 0.0,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.1671159029649596,
      "brier": 0.3670596395956874,
      "ece": 0.34489191374663086,
      "n": 371,
      "score": 31.021617250673827,
      "score_before_time_limit": 31.021617250673827,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.5121951219512195,
      "interval90_width": 0.28523035230352306,
      "n": 369,
      "nrps": 0.263493167195122,
      "score": 36.825341640243906,
      "score_before_time_limit": 36.825341640243906,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.5007441734417342
     }
    },
    "score": 22.615652963639242
   },
   "capability": 15.208879168420996,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.24067576001278546,
     1.3150072706256521
    ],
    "B": [
     0.17589204086840973,
     1.054886576667163
    ],
    "C": [
     0.1671359444533232,
     0.9131994934900359
    ]
   },
   "cost": {
    "basis": "ESTIMATE (same-family reference, one size up; self-hosted open weights, no bookable listing for this base). pricing_v15.reference() RAISES on google/gemma-4-E2B and on google/gemma-4-e2b - the base is in none of the three frozen tables - and the published page carries no priced row on this base either: the one Gemma-4-E2B entry on the board, system-one-open, is listed not_measured. The row therefore carries its own labelled estimate from the nearest reference the frozen 25 Sep snapshot itself contains, deepinfra:google/gemma-4-E4B-it at USD 0.02 per 1M input and USD 0.10 per 1M output - the SAME Gemma-4 effective-parameter family, one size LARGER, so it is both the nearest available reference and the direction that cannot flatter this row - x this row's own measured tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw. RELEASE-LANE LINE, recorded and not acted on: google/gemma-4-E2B needs either a listing decision or an UNLISTED_BASE_MODELS entry in the frozen table before this row may be published; receipts/COST-ALTERNATIVES-R58.json has the alternatives' arithmetic already done.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.009694333333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00984529451591063
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Gemma 4 E2B (Gemma-4-E2B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,477 of 1,500 answered. Weights devops-thiago/classone-gemma4-e2b (sha cd502aa33d288e67f16937c2e125f2297ab7b4ad), Apache-2.0, public and ungated; code github.com/devops-thiago/class-one, Apache-2.0. ONE DOCUMENTED MAPPING OF OURS, AND IT IS THE ONLY ONE. The author's server keys SCORE probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1'. The UNAMENDED smoke gate REFUSED the first attempt at check score-keys-are-indices, before any sealed item was sent, and the server was torn down with the card verified free. The 1-based convention - which their own schemas.py documents only as 'Probability distribution across each ordered rubric level', without stating the convention - was then established on the AUTHOR'S OWN SERVER with SYNTHETIC requests across every rubric size their own validator allows (2..10): exactly the keys '1'..'n' in order every time, never '0' and never a key above n. It is corroborated independently by their own `score` field being a 1-based probability-weighted mean of the same distribution (1.0185 on a 2-level rubric, 1.6147 on a 10-level one, 3.1155 where the mass sits on key 3; a 0-based convention would have given 0.0185), and a reversed rubric moved the mass to a different key, so 'k' means the k-th level of the criteria array AS SENT - which is what JevBench's 'k-1' means. classone_reindex_r58.ClassOneReindexAdapter therefore relabels score keys k -> k-1 and NOTHING else: the request body, the answer mapping, the noul range check and the choice-must-be-one-of-the-item's-labels rule are the UNCHANGED vendored typesafe adapter's, reused by calling its own run(). An item whose score key set is not exactly {'1'..'n'} is REFUSED and recorded invalid rather than rescored on a guess; none was. choice and noul are untouched. The gate then ran AMENDED in exactly that one check, requiring the key set to be exactly {'1'..'n'} - stricter than the original, because it also pins the count - with the other seven byte-identical, and passed. PREREG-R58-AMENDMENT-1.md (sha256 in PREREG-R58-AMENDMENT-1.md.sha256), receipt PROBE-classone-e2b.json. No file of the author's was edited and no prompt of ours exists. THE SUBMIT FORM CARRIED NO DESCRIPTION FOR EITHER CLASSONE ROW, so everything about how it is run comes from the author's own repository: install line README Quickstart step 1 (`pip install -e \".[dev]\"`, in an isolated venv - torch 2.14.1+cu130, transformers 5.19.0, fastapi 0.142.2), serve line step 3 (`uvicorn classone.server.app:app`), and the checkpoint selected the way their own src/classone/server/app.py reads it (CLASSONE_BASE_MODEL). THEIR SERVER REGISTERS POST /v1/systemone ITSELF (app.py:115, next to /v1/decide and /v1/classone) and their DecisionRequest/DecisionResponse in src/classone/schemas.py are the TypeSafe System One shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. output_tokens is 0 by their own schema comment ('In System 1 models this is 0 because there is no autoregressive decode'). THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM at a limit far below them: all 23 came back as 'HTTP 400: Prompt length (81338 tokens) exceeds maximum limit (8192)', an 8,192-token ceiling the author's code enforces with an explicit error rather than truncating. The 23 invalid are the system's own refusal, counted wrong, and no other item failed. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 58, benchmarkheaven.com/submit ref C4D7AACD (bh-submit 4567a065-e912-40df-9bc1-68fb37cc7bba)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB",
   "intelligence": {
    "I_open": 8.900914683249555,
    "I_sealed": 6.703296063155943,
    "base": 7.802105373202749,
    "excess": -0.3757690223446315,
    "gap": 2.1976186200936123,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 26.702744049748667,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 18.270270270270274,
       "hard": 16.955017301038072,
       "judge": 28.80685563612393,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -13.043478260869568,
       "judge": -40.0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.08424242424244,
       "hard": -7.722870370370338,
       "judge": -13.495593220338996,
       "standard": -43.26740740740745
      }
     },
     "sealed|choice": {
      "cc": 20.109888189467828,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 30.82551178774667,
       "hard": 12.952488687782791,
       "judge": 20.856610800744896,
       "standard": 27.946791476782867
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 15.999999999999993,
       "hard": -37.86407766990292,
       "judge": 2.6315789473684292,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 14.937446808510646,
       "hard": -32.15683823529416,
       "judge": -35.42389221556888,
       "standard": -33.18258883248737
      }
     }
    }
   },
   "jevbench_score": 0.49345791688754753,
   "key": "classone-gemma4-e2b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-gemma4-e2b@cd502aa33d288e67f16937c2e125f2297ab7b4ad",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5889967637540453,
    "decisive_rate": 0.8328840970350404,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 22.615652963639242,
     "cost": 70.20313863040258,
     "intelligence": 7.802105373202749,
     "speed": 93.2878888255429
    },
    "board_release": "v1.6.43",
    "capability": 15.208879168420996,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 137,
    "capability_rank": 167,
    "composite_ci95": {
     "A": [
      0.24070026102449685,
      1.3152079969468897
     ],
     "B": [
      0.17590250514236588,
      1.0549899064852435
     ],
     "C": [
      0.16715295904478944,
      0.9133388867686731
     ]
    },
    "jevbench_score": 0.49351800738768026,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 163,
    "rank": 206,
    "ranks": {
     "A": 206,
     "B": 204,
     "C": 206,
     "capability": 239
    },
    "scores": {
     "A": 0.49351800738768026,
     "B": 0.37400131309297324,
     "C": 0.34272083846366685
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20868116198689676,
     "p50_s_raw": 0.029340580993448384,
     "p95_s_adjusted": 0.2247645789931994,
     "p95_s_raw": 0.0373822894965997
    },
    "views": {
     "Balanced 33:33:33": 0.4770150324027566,
     "Emphasis on Accuracy 60:20:20": 0.2973210912399719,
     "Emphasis on Cost 20:20:60": 0.6703202939761591,
     "Emphasis on Speed 20:60:20": 0.6973896308140026,
     "Intelligence only": 0.1899745505796208
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 229,
   "ranked": true,
   "ranks": {
    "A": 229,
    "B": 227,
    "C": 229,
    "capability": 261
   },
   "repo": "https://huggingface.co/devops-thiago/classone-gemma4-e2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.49345791688754753,
    "B": 0.3739737036990429,
    "C": 0.3426791089496858
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20895755998790264,
    "p50_s_raw": 0.02947877999395132,
    "p95_s_adjusted": 0.23551616103213746,
    "p95_s_raw": 0.04275808051606873
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-E2B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      19.456713582756297,
      25.97574684722229
     ],
     "I": [
      5.95996205912109,
      11.37620576911391
     ],
     "capability": [
      13.386066785667875,
      17.91536010233214
     ],
     "composite": {
      "A": [
       0.24067576001278546,
       1.3150072706256521
      ],
      "B": [
       0.17589204086840973,
       1.054886576667163
      ],
      "C": [
       0.1671359444533232,
       0.9131994934900359
      ]
     },
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 22.615652963639242,
     "capability": 15.208879168420996,
     "intelligence": 7.802105373202749
    },
    "per_type": {
     "choice": {
      "accuracy": 0.38266666666666665,
      "calibration": 0.0,
      "cc": 21.051883709254156,
      "cc_by_type": {
       "choice": 21.051883709254156
      },
      "cc_tiered": 23.40631611960825,
      "cc_tiered_open": 26.702744049748667,
      "cc_tiered_sealed": 20.109888189467828,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 21.051883709254156
     },
     "noul": {
      "accuracy": 0.48533333333333334,
      "calibration": 31.021617250673827,
      "cc": -2.9333333333333322,
      "cc_by_type": {
       "noul": -2.9333333333333322
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2853333333333333,
      "calibration": 36.825341640243906,
      "cc": -21.676596491227972,
      "cc_by_type": {
       "score": -21.676596491227972
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 26.647352805544053,
     "capability": 18.967337110684245,
     "intelligence": 11.287321415824435
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.4769401831601685,
    "Emphasis on Accuracy 60:20:20": 0.2973036423168658,
    "Emphasis on Cost 20:20:60": 0.6702316090202144,
    "Emphasis on Speed 20:60:20": 0.6971017348444999,
    "Intelligence only": 0.1899745505796208
   }
  },
  {
   "adapter_id": "images/mghafiri-qwen3.5-0.8b-decision-model/spec.json:mghafiri_adapter.JevLite",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mourad Ghafiri",
   "axes": {
    "calibration": 77.95657740696296,
    "cost": 79.5180630553356,
    "intelligence": 7.297270616183145,
    "speed": 55.90747410573378
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16652875000000003,
      "ece_score": 66.69425,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 66.69425
     },
     "noul": {
      "abstention_rate": 0.7813333333333333,
      "brier": 0.20344673173333316,
      "ece": 0.05893599999999998,
      "n": 375,
      "score": 88.2128
     },
     "score": {
      "interval90_coverage": 0.8586666666666667,
      "interval90_width": 0.6119999999999997,
      "n": 375,
      "nrps": 0.16765142224888893,
      "score": 78.9626822208889,
      "top_ece": 0.12654746666666666
     }
    },
    "score": 77.95657740696296
   },
   "capability": 42.62692401157305,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.18025356076996352,
     1.2178442350406786
    ],
    "B": [
     0.12404410707015784,
     0.9011760636632091
    ],
    "C": [
     0.1251760838680302,
     0.8457251632226935
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00481649630541872
   },
   "decision_limit": null,
   "display": "Qwen3.5-0.8B Decision Model (Mourad Ghafiri)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 4.79058394567308,
    "I_sealed": 9.803957286693208,
    "base": 7.297270616183145,
    "excess": -7.5867609834583725,
    "gap": -5.013373341020128,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 13.339251237806783,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -1.1891891891891837,
       "hard": 13.180245360176167,
       "judge": 20.896506262359924,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -73.91304347826086,
       "judge": -90,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 1.0325005992124563,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 40.642727272727264,
       "hard": -20.778611111111122,
       "judge": 25.48779661016949,
       "standard": -11.83333333333334
      }
     },
     "sealed|choice": {
      "cc": 21.186916712409108,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 24.174118690414627,
       "hard": 20.079185520361975,
       "judge": 20.856610800744896,
       "standard": 22.404236974996934
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -60,
       "hard": -68.93203883495146,
       "judge": -57.89473684210527,
       "standard": -43.66197183098591
      }
     },
     "sealed|score": {
      "cc": 8.224955147670517,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 42.56900709219856,
       "hard": 1.4315441176470767,
       "judge": 8.896227544910184,
       "standard": 3.6328426395938807
      }
     }
    }
   },
   "jevbench_score": 0.47247471382875306,
   "key": "mghafiri-qwen3.5-0.8b-decision-model",
   "last_measured_on": "2026-10-02",
   "licence": "MIT code and training data; Apache-2.0 model weights",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.6",
   "model_pin": null,
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": 0.8780487804878049,
    "decisive_rate": 0.21866666666666668,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.95657740696296,
     "cost": 79.5180630553356,
     "intelligence": 7.297270616183145,
     "speed": 55.90747410573378
    },
    "board_release": "v1.6.43",
    "capability": 42.62692401157305,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.18025356076996352,
      1.2178442350406786
     ],
     "B": [
      0.12404410707015784,
      0.9011760636632091
     ],
     "C": [
      0.1251760838680302,
      0.8457251632226935
     ]
    },
    "jevbench_score": 0.47247471382875306,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 164,
    "rank": 207,
    "ranks": {
     "A": 207,
     "B": 207,
     "C": 207,
     "capability": 200
    },
    "scores": {
     "A": 0.47247471382875306,
     "B": 0.3355765591935497,
     "C": 0.3281074401588563
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 11.252091223932803,
     "p50_s_raw": 5.551045611966401,
     "p95_s_adjusted": 22.804436485376208,
     "p95_s_raw": 11.327218242688105
    },
    "views": {
     "Balanced 33:33:33": 0.381493687842595,
     "Emphasis on Accuracy 60:20:20": 0.24118281186608428,
     "Emphasis on Cost 20:20:60": 0.5528132957063066,
     "Emphasis on Speed 20:60:20": 0.5239265172020665,
     "Intelligence only": 0.15543232661348225
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/mghafiri/qwen3.5-0.8B-decision-model",
   "scores": {
    "A": 0.47247471382875306,
    "B": 0.3355765591935497,
    "C": 0.3281074401588563
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 11.252091223932803,
    "p50_s_raw": 5.551045611966401,
    "p95_s_adjusted": 22.804436485376208,
    "p95_s_raw": 11.327218242688105
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-0.8B-Base fine-tuned as a JevLite decision model with per-question calibration",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.68948470123519,
      80.02198153079908
     ],
     "I": [
      5.167522017117285,
      10.33633269787795
     ],
     "capability": [
      38.95895177314971,
      44.525366548573615
     ],
     "composite": {
      "A": [
       0.18025356076996352,
       1.2178442350406786
      ],
      "B": [
       0.12404410707015784,
       0.9011760636632091
      ],
      "C": [
       0.1251760838680302,
       0.8457251632226935
      ]
     },
     "seed": 67
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.95657740696296,
     "capability": 42.62692401157305,
     "intelligence": 7.297270616183145
    },
    "per_type": {
     "choice": {
      "accuracy": 0.36933333333333335,
      "calibration": 66.69425,
      "cc": 19.346740808806086,
      "cc_by_type": {
       "choice": 19.346740808806086
      },
      "cc_tiered": 17.263083975107946,
      "cc_tiered_open": 13.339251237806783,
      "cc_tiered_sealed": 21.186916712409108,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.346740808806086
     },
     "noul": {
      "accuracy": 0.192,
      "calibration": 88.2128,
      "cc": -61.6,
      "cc_by_type": {
       "noul": -61.6
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4106666666666667,
      "calibration": 78.9626822208889,
      "cc": 7.9552982456141175,
      "cc_by_type": {
       "score": 7.9552982456141175
      },
      "cc_tiered": 4.628727873441487,
      "cc_tiered_open": 1.0325005992124563,
      "cc_tiered_sealed": 8.224955147670517,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 7.9552982456141175
     }
    },
    "raw_on_A": {
     "calibration": 67.24894909368233,
     "capability": 38.149863284895126,
     "intelligence": 9.050777476107926
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.381493687842595,
    "Emphasis on Accuracy 60:20:20": 0.24118281186608428,
    "Emphasis on Cost 20:20:60": 0.5528132957063066,
    "Emphasis on Speed 20:60:20": 0.5239265172020665,
    "Intelligence only": 0.15543232661348225
   }
  },
  {
   "addendum": null,
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Fastino",
   "axes": {
    "calibration": 62.13288114639033,
    "cost": 45.81558630529788,
    "intelligence": 7.011351364482595,
    "speed": 84.98447762423095
   },
   "calibration": {
    "parts": {
     "choice": {
      "basis": "provider_confidence_only",
      "ece_score": 28.328881967263143,
      "n": 305,
      "n_ece": 305,
      "score": 28.328881967263143,
      "score_before_time_limit": 28.328881967263143,
      "time_limit_failed_share": 0.0,
      "top_label_ece": 0.3583555901636843
     },
     "noul": {
      "basis": "provider_confidence_only",
      "ece_score": 76.72217340082736,
      "n": 370,
      "n_ece": 370,
      "score": 76.72217340082736,
      "score_before_time_limit": 76.72217340082736,
      "time_limit_failed_share": 0.0,
      "top_label_ece": 0.11638913299586323
     },
     "score": {
      "basis": "provider_confidence_only",
      "ece_score": 81.3475880710805,
      "n": 366,
      "n_ece": 366,
      "score": 81.3475880710805,
      "score_before_time_limit": 81.3475880710805,
      "time_limit_failed_share": 0.0,
      "top_label_ece": 0.09326205964459748
     }
    },
    "score": 62.13288114639033
   },
   "capability": 34.57211625543646,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     0.10778599179976782,
     1.0396621480462107
    ],
    "B": [
     0.07441782713955666,
     0.785906717748602
    ],
    "C": [
     0.07485138319428321,
     0.7219876028098686
    ]
   },
   "cost": {
    "basis": "ESTIMATE: published Fastino sources conflict (USD 0.03/M vs 0.15/M input); conservative higher USD 0.15/M input, 0/M output x measured tokens; no invoice",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.07366349999999999
    },
    "kind": "mixed",
    "usage_estimated_rows": 15,
    "usd_per_1000": 0.06399515910629654
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Fastino GLiNER-2.5-Decide (hosted API)",
   "endpoint_condition": "external hosted API (Fastino); received the full set: 1,200 sealed + 300 public items (v1.6.1)",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.2445368778313695,
    "I_sealed": 7.778165851133819,
    "base": 7.011351364482595,
    "excess": -4.107016615740694,
    "gap": -1.5336289733024495,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 18.73361063349411,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 18.270270270270274,
       "hard": 15.06763133060712,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -21.739130434782606,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -36.36363636363635,
       "hard": -28.70370370370372,
       "judge": -16.949152542372904,
       "standard": -99.99999999999996
      }
     },
     "sealed|choice": {
      "cc": 15.115509018204584,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 25.504397309881035,
       "hard": 8.880090497737543,
       "judge": 18.06331471135942,
       "standard": 17.970193373568186
      }
     },
     "sealed|noul": {
      "cc": 8.218988535196875,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 4.0000000000000036,
       "hard": -6.796116504854366,
       "judge": 7.8947368421052655,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -38.29787234042554,
       "hard": -44.60784313725492,
       "judge": -24.55089820359284,
       "standard": -63.451776649746215
      }
     }
    }
   },
   "jevbench_score": 0.3433992147410167,
   "key": "fastino-gliner-2-5-decide",
   "last_measured_on": "2026-10-05",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "fastino/GLiNER-2.5-Decide (requested model id)",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5405405405405406,
    "decisive_rate": 1.0,
    "supported": true,
    "valid": 370
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 44,
    "axes": {
     "calibration": 62.13288114639033,
     "cost": 45.81558630529788,
     "intelligence": 7.011351364482595,
     "speed": 84.7052606179839
    },
    "board_release": "v1.6.43",
    "capability": 34.57211625543646,
    "capability_api_board_rank": 28,
    "capability_open_board_rank": null,
    "capability_rank": 151,
    "composite_ci95": {
     "A": [
      0.10777031922591683,
      1.0393833813480395
     ],
     "B": [
      0.07441185007614143,
      0.7857792694163073
     ],
     "C": [
      0.07484049946244224,
      0.7217940148250275
     ]
    },
    "jevbench_score": 0.34332996901846624,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 208,
    "ranks": {
     "A": 208,
     "B": 208,
     "C": 208,
     "capability": 218
    },
    "scores": {
     "A": 0.34332996901846624,
     "B": 0.24643533434670511,
     "C": 0.23842358959615717
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4409300498664379,
     "p50_s_raw": 0.4409300498664379,
     "p95_s_adjusted": 0.7675456959754229,
     "p95_s_raw": 0.7675456959754229
    },
    "views": {
     "Balanced 33:33:33": 0.2810101668608893,
     "Emphasis on Accuracy 60:20:20": 0.17887060474062133,
     "Emphasis on Cost 20:20:60": 0.37538092966437175,
     "Emphasis on Speed 20:60:20": 0.41302239566745963,
     "Intelligence only": 0.11575817604301195
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 230,
   "ranked": true,
   "ranks": {
    "A": 230,
    "B": 230,
    "C": 230,
    "capability": 240
   },
   "repo": "https://fastino.ai/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.3433992147410167,
    "B": 0.2464638726515236,
    "C": 0.23847167690348384
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.42045173048973083,
    "p50_s_raw": 0.42045173048973083,
    "p95_s_adjusted": 0.7548073027282952,
    "p95_s_raw": 0.7548073027282952
   },
   "status": {
    "answered_ok": 1469,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "confidence",
    "noul": "confidence",
    "score": "confidence"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.437755987711,
      66.14780229160992
     ],
     "I": [
      4.625975407661763,
      10.579291699427245
     ],
     "capability": [
      30.439944046543882,
      37.65484132935661
     ],
     "composite": {
      "A": [
       0.10778599179976782,
       1.0396621480462107
      ],
      "B": [
       0.07441782713955666,
       0.785906717748602
      ],
      "C": [
       0.07485138319428321,
       0.7219876028098686
      ]
     },
     "seed": 38
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.13288114639033,
     "capability": 34.57211625543646,
     "intelligence": 7.011351364482595
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3426666666666667,
      "calibration": 28.328881967263143,
      "cc": 15.936455007909936,
      "cc_by_type": {
       "choice": 15.936455007909936
      },
      "cc_tiered": 16.924559825849347,
      "cc_tiered_open": 18.73361063349411,
      "cc_tiered_sealed": 15.115509018204584,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.936455007909936
     },
     "noul": {
      "accuracy": 0.5333333333333333,
      "calibration": 76.72217340082736,
      "cc": 6.666666666666665,
      "cc_by_type": {
       "noul": 6.666666666666665
      },
      "cc_tiered": 4.109494267598437,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 8.218988535196875,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 6.666666666666665
     },
     "score": {
      "accuracy": 0.248,
      "calibration": 81.3475880710805,
      "cc": -42.280701754385944,
      "cc_by_type": {
       "score": -42.280701754385944
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.022666666666666613,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.02400000000000002,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.2810720197974522,
    "Emphasis on Accuracy 60:20:20": 0.17888563915023908,
    "Emphasis on Cost 20:20:60": 0.3754471502388532,
    "Emphasis on Speed 20:60:20": 0.4132629943808129,
    "Intelligence only": 0.11575817604301195
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "LibertAI",
   "axes": {
    "calibration": 29.6278299611924,
    "cost": 80.28784730320886,
    "intelligence": 6.531204174677637,
    "speed": 84.83282651822053
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.5370261325853646,
      "ece_score": 0.0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0.0,
      "score_before_time_limit": 0.0,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2264150943396226,
      "brier": 0.34735578114832993,
      "ece": 0.30478924226600856,
      "n": 371,
      "score": 39.04215154679829,
      "score_before_time_limit": 39.04215154679829,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.6233062330623306,
      "interval90_width": 0.4112466124661248,
      "n": 369,
      "nrps": 0.24237776576192704,
      "score": 49.84133833677892,
      "score_before_time_limit": 49.84133833677892,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.3803977337512473
     }
    },
    "score": 29.6278299611924
   },
   "capability": 18.07951706793502,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     0.12742244350122992,
     0.7721198976940717
    ],
    "B": [
     0.0888613582455493,
     0.5872632921246806
    ],
    "C": [
     0.08848780798696523,
     0.5361943733986607
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate. Frozen market reference deepinfra:Qwen/Qwen3.5-0.8B.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004540166256157636
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Deem 0.8B v1",
   "endpoint_condition": "evaluator-owned H100 GPU",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 8.352332961291632,
    "I_sealed": 4.710075388063642,
    "base": 6.531204174677637,
    "excess": 1.0688699307897456,
    "gap": 3.6422575732279894,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 25.056998883874897,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 41.621621621621614,
       "hard": 9.405473419314262,
       "judge": 40.672379696769944,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0.0,
       "hard": -39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 23.118548982848488,
       "hard": -12.673038757528188,
       "judge": 7.843499385378882,
       "standard": -67.26014388289686
      }
     },
     "sealed|choice": {
      "cc": 14.130226164190928,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 32.15579040721308,
       "hard": 3.7895927601809762,
       "judge": 17.132216014897597,
       "standard": 21.29572607463975
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -39.80582524271845,
       "judge": -2.631578947368418,
       "standard": 4.225352112676051
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -7.688289022450467,
       "hard": -26.74875998927384,
       "judge": -33.081610869032005,
       "standard": -11.930482263268317
      }
     }
    }
   },
   "jevbench_score": 0.323299787357439,
   "key": "deem-0.8-v1",
   "last_measured_on": "2026-10-04",
   "licence": "Not independently recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5923344947735192,
    "decisive_rate": 0.7735849056603774,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 29.6278299611924,
     "cost": 80.28784730320886,
     "intelligence": 6.531204174677637,
     "speed": 85.44505463939521
    },
    "board_release": "v1.6.43",
    "capability": 18.07951706793502,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 135,
    "capability_rank": 165,
    "composite_ci95": {
     "A": [
      0.12746211676468558,
      0.7725020253140645
     ],
     "B": [
      0.08887678609852033,
      0.587439805908629
     ],
     "C": [
      0.08851535886436501,
      0.5364597398014336
     ]
    },
    "jevbench_score": 0.32342918922131697,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 165,
    "rank": 209,
    "ranks": {
     "A": 209,
     "B": 209,
     "C": 209,
     "capability": 237
    },
    "scores": {
     "A": 0.32342918922131697,
     "B": 0.23429153017801835,
     "C": 0.2246036036259146
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4647796718403697,
     "p50_s_raw": 0.15738983592018485,
     "p95_s_adjusted": 0.6141117558348923,
     "p95_s_raw": 0.23205587791744614
    },
    "views": {
     "Balanced 33:33:33": 0.28875743313428337,
     "Emphasis on Accuracy 60:20:20": 0.17645226737338762,
     "Emphasis on Cost 20:20:60": 0.4219666221718261,
     "Emphasis on Speed 20:60:20": 0.4251280933682276,
     "Intelligence only": 0.11143965867359962
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 231,
   "ranked": true,
   "ranks": {
    "A": 231,
    "B": 231,
    "C": 231,
    "capability": 259
   },
   "repo": "https://huggingface.co/LibertAIDAI/deem-0.8-v1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.323299787357439,
    "B": 0.23423719784666683,
    "C": 0.22451374122044376
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4788308489136398,
    "p50_s_raw": 0.1644154244568199,
    "p95_s_adjusted": 0.6863335451111197,
    "p95_s_raw": 0.2681667725555598
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "LibertAIDAI/deem-0.8-v1 @ 8cbabbb2c4a7ef13c6b43f0ef3ae4157983c6d21; root model.safetensors SHA-256 80438125177c0681e855158d46720cbbd4ff07ea620cd55b0f456aafbfcf778d. Author Python GPU server, default DEEM_N_ORDERS=1, raw softmax (temperature 1.0). A four-field shim translates the wire format. The submitted interface accepts choice labels without descriptions and noul instructions without criteria; score receives full level descriptions. The author Rust/AVX-512 latency claim was not measured.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      25.404113930608833,
      32.82152927572935
     ],
     "I": [
      4.660400387407388,
      9.084320195556709
     ],
     "capability": [
      15.752622734868103,
      20.176991128075684
     ],
     "composite": {
      "A": [
       0.12742244350122992,
       0.7721198976940717
      ],
      "B": [
       0.0888613582455493,
       0.5872632921246806
      ],
      "C": [
       0.08848780798696523,
       0.5361943733986607
      ]
     },
     "seed": 35
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 29.6278299611924,
     "capability": 18.07951706793502,
     "intelligence": 6.531204174677637
    },
    "per_type": {
     "choice": {
      "accuracy": 0.34933333333333333,
      "calibration": 0.0,
      "cc": 16.78902645813397,
      "cc_by_type": {
       "choice": 16.78902645813397
      },
      "cc_tiered": 19.593612524032913,
      "cc_tiered_open": 25.056998883874897,
      "cc_tiered_sealed": 14.130226164190928,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 16.78902645813397
     },
     "noul": {
      "accuracy": 0.4533333333333333,
      "calibration": 39.04215154679829,
      "cc": -9.333333333333337,
      "cc_by_type": {
       "noul": -9.333333333333337
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.30933333333333335,
      "calibration": 49.84133833677892,
      "cc": -19.150447666018742,
      "cc_by_type": {
       "score": -19.150447666018742
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 29.6549349776408,
     "capability": 17.995438687555687,
     "intelligence": 6.335942397470573
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.2886199168569223,
    "Emphasis on Accuracy 60:20:20": 0.1764214479496997,
    "Emphasis on Cost 20:20:60": 0.4217904161709171,
    "Emphasis on Speed 20:60:20": 0.42459197741398275,
    "Intelligence only": 0.11143965867359962
   }
  },
  {
   "adapter_id": "images/decision-fast/spec.json:flymy_v15.FlyMyPackage",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "FlyMy.AI (@denti)",
   "axes": {
    "calibration": 75.07737238688318,
    "cost": 80.08651867745036,
    "intelligence": 6.110691078442272,
    "speed": 91.47223870429099
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1702515239808779,
      "ece_score": 65.94969520382442,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 65.94969520382442,
      "score_before_time_limit": 65.94969520382442,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.8625336927223719,
      "brier": 0.2408499885802099,
      "ece": 0.11364094419260821,
      "n": 371,
      "score": 77.27181116147835,
      "score_before_time_limit": 77.27181116147835,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.959349593495935,
      "interval90_width": 0.8803071364046972,
      "n": 369,
      "nrps": 0.17723101114191298,
      "score": 82.01061079534679,
      "score_before_time_limit": 82.01061079534679,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09127838647557562
     }
    },
    "score": 75.07737238688318
   },
   "capability": 40.59403173266273,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.11705708913499598,
     0.8235357256896713
    ],
    "B": [
     0.0785738132381991,
     0.5860130115895594
    ],
    "C": [
     0.0812896452326361,
     0.5718998095067163
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004610868226600985
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision Fast (FlyMy.AI, v53a)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 5.162912545807329,
    "I_sealed": 7.058469611077215,
    "base": 6.110691078442272,
    "excess": -4.468944707708131,
    "gap": -1.8955570652698865,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 15.371451872027388,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 3.743315508021402,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -100.0,
       "judge": -90.0,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0.1172857653946,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 37.515372511743564,
       "hard": -11.188275382721603,
       "judge": -4.115171438893084,
       "standard": 10.37805049488405
      }
     },
     "sealed|choice": {
      "cc": 21.175408833231646,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 32.15579040721308,
       "hard": 4.298642533936631,
       "judge": 27.374301675977662,
       "standard": 40.14041138071193
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -16.000000000000004,
       "hard": -88.3495145631068,
       "judge": -86.8421052631579,
       "standard": -83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 22.391298609400668,
       "hard": -12.696561777971827,
       "judge": 3.397122091897453,
       "standard": -2.490577451116116
      }
     }
    }
   },
   "jevbench_score": 0.2981489679395777,
   "key": "decision-fast",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9019607843137255,
    "decisive_rate": 0.13746630727762804,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 75.07737238688318,
     "cost": 80.08651867745036,
     "intelligence": 6.110691078442272,
     "speed": 91.472185793536
    },
    "board_release": "v1.6.43",
    "capability": 40.59403173266273,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 112,
    "capability_rank": 140,
    "composite_ci95": {
     "A": [
      0.11705708633042304,
      0.8235356910346963
     ],
     "B": [
      0.07857381222727884,
      0.5860129975515587
     ],
     "C": [
      0.08128964328501598,
      0.5718997854407614
     ]
    },
    "jevbench_score": 0.2981489585308357,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 166,
    "rank": 210,
    "ranks": {
     "A": 210,
     "B": 210,
     "C": 210,
     "capability": 204
    },
    "scores": {
     "A": 0.2981489585308357,
     "B": 0.20514889729513758,
     "C": 0.2070478878686359
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2653681320138276,
     "p50_s_raw": 0.05768406600691378,
     "p95_s_adjusted": 0.26849280574824663,
     "p95_s_raw": 0.059246402874123305
    },
    "views": {
     "Balanced 33:33:33": 0.23953342944478384,
     "Emphasis on Accuracy 60:20:20": 0.14519180403822335,
     "Emphasis on Cost 20:20:60": 0.3522038654235207,
     "Emphasis on Speed 20:60:20": 0.35744386580296805,
     "Intelligence only": 0.09127061519323529
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 232,
   "ranked": true,
   "ranks": {
    "A": 232,
    "B": 232,
    "C": 232,
    "capability": 227
   },
   "repo": "https://huggingface.co/flymy-ai/decision-fast-preview",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.2981489679395777,
    "B": 0.20514890085877344,
    "C": 0.20704789440248456
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2652455839328468,
    "p50_s_raw": 0.05762279196642339,
    "p95_s_adjusted": 0.2686135815223679,
    "p95_s_raw": 0.05930679076118395
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3-0.6B-Base with a trained LoRA adapter and pointer head (10.6M trainable parameters)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      69.51262824417705,
      78.1703871756354
     ],
     "I": [
      4.394274632210615,
      8.794694293942209
     ],
     "capability": [
      37.53859057048145,
      42.929975748521436
     ],
     "composite": {
      "A": [
       0.11705708913499598,
       0.8235357256896713
      ],
      "B": [
       0.0785738132381991,
       0.5860130115895594
      ],
      "C": [
       0.0812896452326361,
       0.5718998095067163
      ]
     },
     "seed": 32
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.07737238688318,
     "capability": 40.59403173266273,
     "intelligence": 6.110691078442272
    },
    "per_type": {
     "choice": {
      "accuracy": 0.37333333333333335,
      "calibration": 65.94969520382442,
      "cc": 19.85828367894051,
      "cc_by_type": {
       "choice": 19.85828367894051
      },
      "cc_tiered": 18.273430352629518,
      "cc_tiered_open": 15.371451872027388,
      "cc_tiered_sealed": 21.175408833231646,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.85828367894051
     },
     "noul": {
      "accuracy": 0.12266666666666666,
      "calibration": 77.27181116147835,
      "cc": -75.46666666666667,
      "cc_by_type": {
       "noul": -75.46666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.31733333333333336,
      "calibration": 82.01061079534679,
      "cc": -1.6484760263063114,
      "cc_by_type": {
       "score": -1.6484760263063114
      },
      "cc_tiered": 0.0586428826973,
      "cc_tiered_open": 0.1172857653946,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 74.21190941158237,
     "capability": 40.84165753605771,
     "intelligence": 7.471405660533052
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.23953343754200446,
    "Emphasis on Accuracy 60:20:20": 0.14519180582322713,
    "Emphasis on Cost 20:20:60": 0.35220387592724206,
    "Emphasis on Speed 20:60:20": 0.35744389825873907,
    "Intelligence only": 0.09127061519323529
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 73.02134054125926,
    "cost": 89.11296503596958,
    "intelligence": 6.054813340422714,
    "speed": 94.88950688832834
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1464534374999999,
      "ece_score": 70.70931250000002,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 70.70931250000002,
      "score_before_time_limit": 70.70931250000002,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.72,
      "brier": 0.2701432261600001,
      "ece": 0.15907813333333334,
      "n": 375,
      "score": 68.18437333333334,
      "score_before_time_limit": 68.18437333333334,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9653333333333334,
      "interval90_width": 0.8146666666666667,
      "n": 375,
      "nrps": 0.1639986175244445,
      "score": 80.17033579044444,
      "score_before_time_limit": 80.17033579044444,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.11629733333333336
     }
    },
    "score": 73.02134054125926
   },
   "capability": 39.53807694084099,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.05507555671906829,
     0.8570652295031572
    ],
    "B": [
     0.036403719413010745,
     0.6085722574185467
    ],
    "C": [
     0.038246914388241855,
     0.5951841871549702
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (ModernBERT-base has no market listing; base-size encoder reference) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0023062222906403943
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Small-EN (Chinh Nguyen, ModernBERT-base encoder with fixed heads)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 5.966953908358329,
    "I_sealed": 6.142672772487099,
    "base": 6.054813340422714,
    "excess": -2.749106506567014,
    "gap": -0.17571886412877014,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 11.596876448581234,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 24.50456118276188,
       "judge": 1.1206328279499007,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -65.21739130434783,
       "judge": -70.0,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 6.303985276493753,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 12.853939393939395,
       "hard": 15.763981481481492,
       "judge": 4.772711864406753,
       "standard": -13.594074074074047
      }
     },
     "sealed|choice": {
      "cc": 14.421677328746334,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 4.219939398418472,
       "hard": 11.934389140271476,
       "judge": 14.338919925512121,
       "standard": 24.621258775711308
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -48.0,
       "hard": -84.46601941747574,
       "judge": -52.63157894736843,
       "standard": -60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 4.006340988714965,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 17.547730496453894,
       "hard": 1.5428921568626852,
       "judge": 5.2876646706586765,
       "standard": 0.240558375634492
      }
     }
    }
   },
   "jevbench_score": 0.2923902780429571,
   "key": "quyet-1-0-small-en",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.638095238095238,
    "decisive_rate": 0.28,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.02134054125926,
     "cost": 89.11296503596958,
     "intelligence": 6.054813340422714,
     "speed": 94.90893356488051
    },
    "board_release": "v1.6.43",
    "capability": 39.53807694084099,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 114,
    "capability_rank": 142,
    "composite_ci95": {
     "A": [
      0.0550759146094291,
      0.8570777384335173
     ],
     "B": [
      0.03640384449772412,
      0.6085773029152636
     ],
     "C": [
      0.03824716292321466,
      0.5951928739121649
     ]
    },
    "jevbench_score": 0.29239342204088814,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 167,
    "rank": 211,
    "ranks": {
     "A": 211,
     "B": 211,
     "C": 211,
     "capability": 206
    },
    "scores": {
     "A": 0.29239342204088814,
     "B": 0.20045885649425463,
     "C": 0.20305098752839457
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1791461439919658,
     "p50_s_raw": 0.0145730719959829,
     "p95_s_adjusted": 0.18025992200709878,
     "p95_s_raw": 0.015129961003549396
    },
    "views": {
     "Balanced 33:33:33": 0.23536198594902974,
     "Emphasis on Accuracy 60:20:20": 0.14175760897818884,
     "Emphasis on Cost 20:20:60": 0.3502184575801078,
     "Emphasis on Speed 20:60:20": 0.3525263048935194,
     "Intelligence only": 0.08878963459745969
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 233,
   "ranked": true,
   "ranks": {
    "A": 233,
    "B": 233,
    "C": 233,
    "capability": 229
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small-EN",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.2923902780429571,
    "B": 0.20045767429781194,
    "C": 0.203048804196498
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.17937943699362222,
    "p50_s_raw": 0.01468971849681111,
    "p95_s_adjusted": 0.18083257000980665,
    "p95_s_raw": 0.01541628500490333
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "answerdotai/ModernBERT-base",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.50774946138796,
      76.70202320961018
     ],
     "I": [
      3.379327510519746,
      8.897820124597144
     ],
     "capability": [
      36.01781590414684,
      42.01829735190088
     ],
     "composite": {
      "A": [
       0.05507555671906829,
       0.8570652295031572
      ],
      "B": [
       0.036403719413010745,
       0.6085722574185467
      ],
      "C": [
       0.038246914388241855,
       0.5951841871549702
      ]
     },
     "seed": 89
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.02134054125926,
     "capability": 39.53807694084099,
     "intelligence": 6.054813340422714
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3253333333333333,
      "calibration": 70.70931250000002,
      "cc": 13.719769237327435,
      "cc_by_type": {
       "choice": 13.719769237327435
      },
      "cc_tiered": 13.009276888663784,
      "cc_tiered_open": 11.596876448581234,
      "cc_tiered_sealed": 14.421677328746334,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.719769237327435
     },
     "noul": {
      "accuracy": 0.17866666666666667,
      "calibration": 68.18437333333334,
      "cc": -64.26666666666667,
      "cc_by_type": {
       "noul": -64.26666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.312,
      "calibration": 80.17033579044444,
      "cc": 5.329298245614,
      "cc_by_type": {
       "score": 5.329298245614
      },
      "cc_tiered": 5.155163132604359,
      "cc_tiered_open": 6.303985276493753,
      "cc_tiered_sealed": 4.006340988714965,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 5.329298245614
     }
    },
    "raw_on_A": {
     "calibration": 75.9241654666809,
     "capability": 42.17966881905028,
     "intelligence": 8.435172171419651
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.23535926977189042,
    "Emphasis on Accuracy 60:20:20": 0.14175701778059244,
    "Emphasis on Cost 20:20:60": 0.35021484917491197,
    "Emphasis on Speed 20:60:20": 0.35251533676546687,
    "Intelligence only": 0.08878963459745969
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Convai Innovations",
   "axes": {
    "calibration": 82.96996460403702,
    "cost": 82.52942826196218,
    "intelligence": 5.471270906587037,
    "speed": 69.34079458737523
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08964031250000004,
      "ece_score": 82.07193749999999,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 82.07193749999999,
      "score_before_time_limit": 82.07193749999999,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.9946666666666667,
      "brier": 0.24736279077333312,
      "ece": 0.09575093333333361,
      "n": 375,
      "score": 80.84981333333327,
      "score_before_time_limit": 80.84981333333327,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.968,
      "interval90_width": 0.8166666666666665,
      "n": 375,
      "nrps": 0.15890220709111097,
      "score": 85.9881429787778,
      "score_before_time_limit": 85.9881429787778,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.060667466666666586
     }
    },
    "score": 82.96996460403702
   },
   "capability": 44.22061775531203,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     0.038429115573591126,
     0.5887972663831158
    ],
    "B": [
     0.02532188423731967,
     0.41619886251400867
    ],
    "C": [
     0.02668688581499384,
     0.4088869905438304
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003822543103448276
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Laya typed-decisions",
   "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 5.6928563402143615,
    "I_sealed": 5.249685472959713,
    "base": 5.471270906587037,
    "excess": -2.130216775183595,
    "gap": 0.4431708672546488,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 10.771453572375108,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": -1.9188424032714584,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 6.307115448267977,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 5.935454545454554,
       "hard": -3.607592592592601,
       "judge": 9.858813559322,
       "standard": 20.99481481481481
      }
     },
     "sealed|choice": {
      "cc": 7.102994687643811,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 5.550218017884884,
       "hard": 2.771493212669664,
       "judge": 6.890130353817524,
       "standard": 16.861682473210998
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -96.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "sealed|score": {
      "cc": 8.646061731235328,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 9.860354609929068,
       "hard": 2.728602941176461,
       "judge": 13.044251497005966,
       "standard": 13.276548223350238
      }
     }
    }
   },
   "jevbench_score": 0.21636634154218096,
   "key": "laya-typed-decisions",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 1.0,
    "decisive_rate": 0.005333333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.96996460403702,
     "cost": 82.52942826196218,
     "intelligence": 5.471270906587037,
     "speed": 70.81965680870755
    },
    "board_release": "v1.6.43",
    "capability": 44.22061775531203,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.03846019800643322,
      0.5898651799347397
     ],
     "B": [
      0.025332676441949146,
      0.41662539701360535
     ],
     "C": [
      0.02670847083780085,
      0.40962859717690253
     ]
    },
    "jevbench_score": 0.21666109526429145,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 170,
    "rank": 214,
    "ranks": {
     "A": 214,
     "B": 214,
     "C": 214,
     "capability": 190
    },
    "scores": {
     "A": 0.21666109526429145,
     "B": 0.14825243435052415,
     "C": 0.1504590939335357
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.8777340129803632,
     "p50_s_raw": 0.8638670064901817,
     "p95_s_adjusted": 4.409610662701016,
     "p95_s_raw": 2.1298053313505076
    },
    "views": {
     "Balanced 33:33:33": 0.17186613033097944,
     "Emphasis on Accuracy 60:20:20": 0.10420153536931118,
     "Emphasis on Cost 20:20:60": 0.2566823909067557,
     "Emphasis on Speed 20:60:20": 0.2523472425383811,
     "Intelligence only": 0.06551257180569732
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 235,
   "ranked": true,
   "ranks": {
    "A": 235,
    "B": 235,
    "C": 235,
    "capability": 211
   },
   "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/typed-decisions",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.21636634154218096,
    "B": 0.1481419607342047,
    "C": 0.15025440384873676
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.1531285819859476,
    "p50_s_raw": 1.0015642909929738,
    "p95_s_adjusted": 5.405683011900692,
    "p95_s_raw": 2.6278415059503457
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, typed-decisions/; ModernBERT-large, max_len 1024, distinct head and calibration from the English root. Fresh full run on 29 Sep; in-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. Complete valid probabilities on all types; frequent noul abstention lowers Intelligence to its zero floor and thus the composite to zero. This is not a transport failure.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.7245419121787,
      86.71414939090002
     ],
     "I": [
      2.9916515598721545,
      7.823617519507115
     ],
     "capability": [
      41.079834277143355,
      46.819056265617796
     ],
     "composite": {
      "A": [
       0.038429115573591126,
       0.5887972663831158
      ],
      "B": [
       0.02532188423731967,
       0.41619886251400867
      ],
      "C": [
       0.02668688581499384,
       0.4088869905438304
      ]
     },
     "seed": 58
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.96996460403702,
     "capability": 44.22061775531203,
     "intelligence": 5.471270906587037
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2733333333333333,
      "calibration": 82.07193749999999,
      "cc": 7.069711925579947,
      "cc_by_type": {
       "choice": 7.069711925579947
      },
      "cc_tiered": 8.93722413000946,
      "cc_tiered_open": 10.771453572375108,
      "cc_tiered_sealed": 7.102994687643811,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 7.069711925579947
     },
     "noul": {
      "accuracy": 0.005333333333333333,
      "calibration": 80.84981333333327,
      "cc": -98.93333333333332,
      "cc_by_type": {
       "noul": -98.93333333333332
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.36533333333333334,
      "calibration": 85.9881429787778,
      "cc": 7.23879824561402,
      "cc_by_type": {
       "score": 7.23879824561402
      },
      "cc_tiered": 7.476588589751652,
      "cc_tiered_open": 6.307115448267977,
      "cc_tiered_sealed": 8.646061731235328,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 7.23879824561402
     }
    },
    "raw_on_A": {
     "calibration": 79.92527034876068,
     "capability": 42.6053892600956,
     "intelligence": 5.28550817143052
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.17161885391660758,
    "Emphasis on Accuracy 60:20:20": 0.10414694710182727,
    "Emphasis on Cost 20:20:60": 0.25635140429607584,
    "Emphasis on Speed 20:60:20": 0.25138994625628774,
    "Intelligence only": 0.06551257180569732
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Samrat Dutta (samratduttaofficial@gmail.com)",
   "axes": {
    "calibration": 65.7809648808148,
    "cost": 68.67136467637603,
    "intelligence": 5.305239654184986,
    "speed": 77.39270161387154
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21968406249999994,
      "ece_score": 56.06318750000001,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 56.06318750000001,
      "score_before_time_limit": 56.06318750000001,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.6266666666666667,
      "brier": 0.25745770685333363,
      "ece": 0.15462693333333336,
      "n": 375,
      "score": 68.89041436444444,
      "score_before_time_limit": 69.07461333333333,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.904,
      "interval90_width": 0.7419999999999997,
      "n": 375,
      "nrps": 0.18400134444000008,
      "score": 72.389292778,
      "score_before_time_limit": 72.389292778,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.18410639999999998
     }
    },
    "score": 65.7809648808148
   },
   "capability": 35.5431022674999,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.06778805244371189,
     0.5947334633172188
    ],
    "B": [
     0.04550101039862949,
     0.42548800490265093
    ],
    "C": [
     0.047075036419244376,
     0.41300934952584634
    ]
   },
   "cost": {
    "basis": "ESTIMATE, labelled, and RUN 48'S RATE FOR THIS EXACT ROW AND BASE, CARRIED UNCHANGED: answerdotai/ModernBERT-base IS in the frozen pricing_v15.UNLISTED_BASE_MODELS, so no exact base-model floor applies and the module takes the estimate rate from the row's own registry entry; USD 0.03/M input, 0 output is the rate run 47 (quyet-10-small-en) and run 48 (watersheep on v1.5) already assigned to this base. Output 0 is literal: watersheep/infer.py returns output_tokens 0 and the serving path has no generation step (02 Oct deputy rule). x this row's own measured input tokens. Estimated, not charged. No new decision.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.011139300000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011073574813811781
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "WaterSheep (Samrat Dutta, ModernBERT-base encoder with calibrated heads, served by the author's own `watersheep --serve`)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. RE-MEASUREMENT on v1.6 of the row run 48 measured on the retired v1.5 basis, same weights: samratduttaofficial/WaterSheep @ ebf3705d2d9dc648d5cc6b4cbca7646c6ab48998 (Apache-2.0), package github.com/SamratDuttaOfficial/WaterSheep @ 3ccf02b3 via the author's own install line in an isolated venv (torch 2.14.1+cpu, transformers 5.19.0), served with the author's own command `watersheep --model samratduttaofficial/WaterSheep@ebf3705d... --serve --device cpu`. Their server speaks POST /v1/systemone with Jev's shapes, so the UNCHANGED typesafe adapter applies; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. Their own max_len 512 (watersheep.json) truncates every longer state, the 23 long items included, without refusing; not overridden. RUN ON SANDY'S SHARED CPU (12-core Zen 2, 8 threads) with server and driver inside a loopback-only network namespace and no credential in the environment; load average 8.6-15.4 (median 13.8) logged every 60 s, so the Speed axis is a DECLARED LOWER BOUND. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline. add-requests run 59, benchmarkheaven.com/submit ref 6F1561FC; previously GitHub #179",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "CPU (12-core Zen 2; 8 threads)",
   "intelligence": {
    "I_open": 5.71828897246665,
    "I_sealed": 4.892190335903323,
    "base": 5.305239654184986,
    "excess": -1.7472890058749173,
    "gap": 0.8260986365633265,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 17.15486691739995,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.37837837837838,
       "hard": 11.292859389745214,
       "judge": 20.896506262359924,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0.0,
       "hard": -82.6086956521739,
       "judge": -50.0,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 24.005151515151525,
       "hard": -8.935277777777806,
       "judge": -0.7272881355932403,
       "standard": -31.58629629629628
      }
     },
     "sealed|choice": {
      "cc": 14.67657100770997,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 22.843840070948207,
       "hard": 8.880090497737543,
       "judge": 15.270018621973943,
       "standard": 21.29572607463975
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -7.9999999999999964,
       "hard": -61.165048543689316,
       "judge": -60.526315789473685,
       "standard": -49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 13.4399290780142,
       "hard": -8.180808823529384,
       "judge": 3.6488023952095605,
       "standard": 1.3909137055837406
      }
     }
    }
   },
   "jevbench_score": 0.19479747591837096,
   "key": "watersheep",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "samratduttaofficial/WaterSheep@ebf3705d2d9dc648d5cc6b4cbca7646c6ab48998",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6906474820143885,
    "decisive_rate": 0.3716577540106952,
    "supported": true,
    "valid": 374
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 65.84236453711111,
     "cost": 68.67136467637603,
     "intelligence": 5.305239654184986,
     "speed": 77.5933742155104
    },
    "board_release": "v1.6.43",
    "capability": 35.57380209564805,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 121,
    "capability_rank": 149,
    "composite_ci95": {
     "A": [
      0.06779535705435,
      0.5949014809956202
     ],
     "B": [
      0.04550364299502931,
      0.4255562687724439
     ],
     "C": [
      0.047080109065520834,
      0.4131260284691807
     ]
    },
    "jevbench_score": 0.19483758736149703,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 171,
    "rank": 215,
    "ranks": {
     "A": 215,
     "B": 215,
     "C": 215,
     "capability": 214
    },
    "scores": {
     "A": 0.19483758736149703,
     "B": 0.13414679955920575,
     "C": 0.13530388011215072
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.9529444739222527,
     "p50_s_raw": 0.4014722369611263,
     "p95_s_adjusted": 1.82639614790678,
     "p95_s_raw": 0.8381980739533901
    },
    "views": {
     "Balanced 33:33:33": 0.15640575110581917,
     "Emphasis on Accuracy 60:20:20": 0.09493747584576766,
     "Emphasis on Cost 20:20:60": 0.2296969891012095,
     "Emphasis on Speed 20:60:20": 0.23287926981068977,
     "Intelligence only": 0.05972759292809448
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 237,
   "ranked": true,
   "ranks": {
    "A": 237,
    "B": 237,
    "C": 237,
    "capability": 237
   },
   "repo": "https://huggingface.co/samratduttaofficial/WaterSheep",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.19479747591837096,
    "B": 0.13413158660410493,
    "C": 0.13527602494331317
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.07510986328125,
    "p50_s_raw": 0.462554931640625,
    "p95_s_adjusted": 1.6954187914729117,
    "p95_s_raw": 0.7727093957364559
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "answerdotai/ModernBERT-base",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      60.09500897475834,
      69.81698087786482
     ],
     "I": [
      3.662660743750301,
      7.923346736956256
     ],
     "capability": [
      32.39438635504768,
      38.32689445842991
     ],
     "composite": {
      "A": [
       0.06778805244371189,
       0.5947334633172188
      ],
      "B": [
       0.04550101039862949,
       0.42548800490265093
      ],
      "C": [
       0.047075036419244376,
       0.41300934952584634
      ]
     },
     "seed": 149
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 65.7809648808148,
     "capability": 35.5431022674999,
     "intelligence": 5.305239654184986
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 56.06318750000001,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 15.91571896255496,
      "cc_tiered_open": 17.15486691739995,
      "cc_tiered_sealed": 14.67657100770997,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.256,
      "calibration": 68.89041436444444,
      "cc": -48.8,
      "cc_by_type": {
       "noul": -48.8
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.32266666666666666,
      "calibration": 72.389292778,
      "cc": -1.428035087719337,
      "cc_by_type": {
       "score": -1.428035087719337
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 58.9569335572792,
     "capability": 33.53090791683189,
     "intelligence": 8.104882276384586
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.15638155138191312,
    "Emphasis on Accuracy 60:20:20": 0.09493212559285719,
    "Emphasis on Cost 20:20:60": 0.2296656724860263,
    "Emphasis on Speed 20:60:20": 0.23278272559462182,
    "Intelligence only": 0.05972759292809448
   }
  },
  {
   "adapter_id": "images/openjev-verdict-1.4/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Hemant (heman10x)",
   "axes": {
    "calibration": 75.69890703057408,
    "cost": 86.62313527343974,
    "intelligence": 5.035901218168153,
    "speed": 80.93559275082234
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13155601347182505,
      "ece_score": 73.68879730563499,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 73.68879730563499,
      "score_before_time_limit": 73.68879730563499,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 1.0,
      "brier": 0.2495580281877627,
      "ece": 0.12226520797197521,
      "n": 375,
      "score": 75.54695840560495,
      "score_before_time_limit": 75.54695840560495,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9893333333333333,
      "interval90_width": 0.916888888888889,
      "n": 375,
      "nrps": 0.17500942351683016,
      "score": 77.8609653804823,
      "score_before_time_limit": 77.8609653804823,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1338856344367619
     }
    },
    "score": 75.69890703057408
   },
   "capability": 40.36740412437111,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.037381561171560676,
     0.4670303161269418
    ],
    "B": [
     0.02459567039766373,
     0.3261942564122147
    ],
    "C": [
     0.02595941748025047,
     0.32432660842148725
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.002791871921182266
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "openJev Verdict 1.4",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 5.214507036728864,
    "I_sealed": 4.857295399607441,
    "base": 5.035901218168153,
    "excess": -2.216176005316821,
    "gap": 0.3572116371214227,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 15.643521110186592,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -5.081081081081076,
       "hard": 13.180245360176167,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 11.206824837702234,
       "hard": -8.9038551649125,
       "judge": -0.9107640863542654,
       "standard": -33.62275309182012
      }
     },
     "sealed|choice": {
      "cc": 8.257811980618104,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 5.550218017884884,
       "hard": 6.334841628959255,
       "judge": 9.683426443202995,
       "standard": 11.319127971425068
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "sealed|score": {
      "cc": 6.314074218204219,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 14.960976046364117,
       "hard": 5.800113224444569,
       "judge": 5.914716157590583,
       "standard": 3.617582382564022
      }
     }
    }
   },
   "jevbench_score": 0.17216463673473972,
   "key": "openjev-verdict-1.4",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0.0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 75.69890703057408,
     "cost": 86.62313527343974,
     "intelligence": 5.035901218168153,
     "speed": 81.49274954261848
    },
    "board_release": "v1.6.43",
    "capability": 40.36740412437111,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 113,
    "capability_rank": 141,
    "composite_ci95": {
     "A": [
      0.0373899767734964,
      0.4672540094953807
     ],
     "B": [
      0.024598584677982253,
      0.3262815362283655
     ],
     "C": [
      0.025965261648261392,
      0.3244819510384588
     ]
    },
    "jevbench_score": 0.17222636557331997,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 172,
    "rank": 216,
    "ranks": {
     "A": 216,
     "B": 217,
     "C": 216,
     "capability": 205
    },
    "scores": {
     "A": 0.17222636557331997,
     "B": 0.11682093815444984,
     "C": 0.11960164275925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7352308319881559,
     "p50_s_raw": 0.2926154159940779,
     "p95_s_adjusted": 0.9644980074837803,
     "p95_s_raw": 0.40724900374189016
    },
    "views": {
     "Balanced 33:33:33": 0.13684263027905905,
     "Emphasis on Accuracy 60:20:20": 0.08186844689007573,
     "Emphasis on Cost 20:20:60": 0.2066197471065469,
     "Emphasis on Speed 20:60:20": 0.20540349835976665,
     "Intelligence only": 0.05108478843903597
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 238,
   "ranked": true,
   "ranks": {
    "A": 238,
    "B": 239,
    "C": 238,
    "capability": 228
   },
   "repo": "https://huggingface.co/heman10x/rlcd-modernbert-151m",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.17216463673473972,
    "B": 0.11679821380829648,
    "C": 0.11955877551023596
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8207729119807482,
    "p50_s_raw": 0.3353864559903741,
    "p95_s_adjusted": 0.9822402124293148,
    "p95_s_raw": 0.4161201062146574
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "GLiClass ModernBERT-base fine-tuned decision model, 151M; v1.4 fixed inference engine",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.20079626876355,
      79.44394588474857
     ],
     "I": [
      2.9611688986465445,
      7.176652677169861
     ],
     "capability": [
      37.175911743687855,
      42.54492005899604
     ],
     "composite": {
      "A": [
       0.037381561171560676,
       0.4670303161269418
      ],
      "B": [
       0.02459567039766373,
       0.3261942564122147
      ],
      "C": [
       0.02595941748025047,
       0.32432660842148725
      ]
     },
     "seed": 83
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.69890703057408,
     "capability": 40.36740412437111,
     "intelligence": 5.035901218168153
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2866666666666667,
      "calibration": 73.68879730563499,
      "cc": 8.774854826028024,
      "cc_by_type": {
       "choice": 8.774854826028024
      },
      "cc_tiered": 11.950666545402349,
      "cc_tiered_open": 15.643521110186592,
      "cc_tiered_sealed": 8.257811980618104,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 8.774854826028024
     },
     "noul": {
      "accuracy": 0.0,
      "calibration": 75.54695840560495,
      "cc": -100.0,
      "cc_by_type": {
       "noul": -100.0
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.344,
      "calibration": 77.8609653804823,
      "cc": 4.055284730748376,
      "cc_by_type": {
       "score": 4.055284730748376
      },
      "cc_tiered": 3.1570371091021094,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 6.314074218204219,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 4.055284730748376
     }
    },
    "raw_on_A": {
     "calibration": 73.35601823970862,
     "capability": 39.682189077168275,
     "intelligence": 6.008359914627925
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.13679067123541433,
    "Emphasis on Accuracy 60:20:20": 0.08185728575415545,
    "Emphasis on Cost 20:20:60": 0.2065486701646049,
    "Emphasis on Speed 20:60:20": 0.20519291407630152,
    "Intelligence only": 0.05108478843903597
   }
  },
  {
   "adapter_id": "images/raw-qwen3-0.6b/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 10.94062892769364,
    "cost": 77.58073177551512,
    "intelligence": 5.554864880982027,
    "speed": 93.42809294775304
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.6186890197483605,
      "ece_score": 0.0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0.0,
      "score_before_time_limit": 0.0,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.580040467448705,
      "ece": 0.5809468363625269,
      "n": 371,
      "score": 0.0,
      "score_before_time_limit": 0.0,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.5067750677506775,
      "interval90_width": 0.31887985546522124,
      "n": 369,
      "nrps": 0.34356226433838155,
      "score": 32.82188678308092,
      "score_before_time_limit": 32.82188678308092,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.6266298915392519
     }
    },
    "score": 10.94062892769364
   },
   "capability": 8.247746904337834,
   "class": "raw-logit-control",
   "composite_ci95": {
    "A": [
     0.046485367385849095,
     0.3704072056350183
    ],
    "B": [
     0.03377797390866881,
     0.3017745506514837
    ],
    "C": [
     0.03228150512906187,
     0.25722722613542937
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005588676108374384
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 0.6B direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 6.274289147536377,
    "I_sealed": 4.835440614427676,
    "base": 5.554864880982027,
    "excess": -1.1345391093295425,
    "gap": 1.4388485331087013,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 18.822867442609134,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 9.405473419314262,
       "judge": 28.80685563612393,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.408700317953674,
       "hard": -23.213473292255358,
       "judge": -47.812096684203475,
       "standard": -39.963164880126655
      }
     },
     "sealed|choice": {
      "cc": 14.506321843283029,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 28.164954548813856,
       "hard": -1.8099547511312433,
       "judge": 26.443202979515842,
       "standard": 22.404236974996934
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -24.0,
       "hard": -22.330097087378643,
       "judge": -5.263157894736848,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -8.286778561594211,
       "hard": -56.317892789427006,
       "judge": -50.51708927969312,
       "standard": -75.44853557058413
      }
     }
    }
   },
   "jevbench_score": 0.16734719426045702,
   "key": "raw-qwen3-0.6b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.41778975741239893,
    "decisive_rate": 1.0,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 10.94062892769364,
     "cost": 77.58073177551512,
     "intelligence": 5.554864880982027,
     "speed": 93.52703208725202
    },
    "board_release": "v1.6.43",
    "capability": 8.247746904337834,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 140,
    "capability_rank": 170,
    "composite_ci95": {
     "A": [
      0.04648667178844407,
      0.37042416292233377
     ],
     "B": [
      0.033778521738855774,
      0.30178355486696584
     ],
     "C": [
      0.032282410964197275,
      0.2572390020293985
     ]
    },
    "jevbench_score": 0.1673536172919218,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 173,
    "rank": 217,
    "ranks": {
     "A": 217,
     "B": 216,
     "C": 217,
     "capability": 242
    },
    "scores": {
     "A": 0.1673536172919218,
     "B": 0.12991425235961862,
     "C": 0.11621778978605682
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2079640818759799,
     "p50_s_raw": 0.02898204093798995,
     "p95_s_adjusted": 0.21345604490488768,
     "p95_s_raw": 0.03172802245244384
    },
    "views": {
     "Balanced 33:33:33": 0.181861744313326,
     "Emphasis on Accuracy 60:20:20": 0.10948842934445303,
     "Emphasis on Cost 20:20:60": 0.26903830305770593,
     "Emphasis on Speed 20:60:20": 0.27429427173292537,
     "Intelligence only": 0.06856152826445967
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 239,
   "ranked": true,
   "ranks": {
    "A": 239,
    "B": 238,
    "C": 239,
    "capability": 264
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-0.6B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.16734719426045702,
    "B": 0.12991115579788795,
    "C": 0.11621332934753961
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20884675215929746,
    "p50_s_raw": 0.029423376079648733,
    "p95_s_adjusted": 0.2174517862033099,
    "p95_s_raw": 0.03372589310165495
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-0.6B BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      10.514844290675109,
      11.37896862687295
     ],
     "I": [
      3.4338072843331555,
      7.572845892568445
     ],
     "capability": [
      7.166123426414619,
      9.279855986833727
     ],
     "composite": {
      "A": [
       0.046485367385849095,
       0.3704072056350183
      ],
      "B": [
       0.03377797390866881,
       0.3017745506514837
      ],
      "C": [
       0.03228150512906187,
       0.25722722613542937
      ]
     },
     "seed": 94
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 10.94062892769364,
     "capability": 8.247746904337834,
     "intelligence": 5.554864880982027
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 0.0,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 16.66459464294608,
      "cc_tiered_open": 18.822867442609134,
      "cc_tiered_sealed": 14.506321843283029,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.41333333333333333,
      "calibration": 0.0,
      "cc": -17.333333333333336,
      "cc_by_type": {
       "noul": -17.333333333333336
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.256,
      "calibration": 32.82188678308092,
      "cc": -46.098542643497574,
      "cc_by_type": {
       "score": -46.098542643497574
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 10.73890139109323,
     "capability": 8.515857103448049,
     "intelligence": 6.292812815802869
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.18185163122500353,
    "Emphasis on Accuracy 60:20:20": 0.10948622994443444,
    "Emphasis on Cost 20:20:60": 0.26902502350979207,
    "Emphasis on Speed 20:60:20": 0.27425286550542904,
    "Intelligence only": 0.06856152826445967
   }
  },
  {
   "adapter_id": "images/lev-350m/spec.json:lev_v15.LevAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Franck Verrot (franckverrot)",
   "axes": {
    "calibration": 81.86424479313416,
    "cost": 80.03797760430007,
    "intelligence": 4.827562367090417,
    "speed": 93.7625770756901
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04337133550488594,
      "ece_score": 91.32573289902281,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 91.32573289902281,
      "score_before_time_limit": 91.32573289902281,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.921832884097035,
      "brier": 0.2643957088948787,
      "ece": 0.12656064690026958,
      "n": 371,
      "score": 74.68787061994608,
      "score_before_time_limit": 74.68787061994608,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.978319783197832,
      "interval90_width": 0.9372177055103887,
      "n": 369,
      "nrps": 0.17965857520325207,
      "score": 79.5791308604336,
      "score_before_time_limit": 79.5791308604336,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.11437940379403796
     }
    },
    "score": 81.86424479313416
   },
   "capability": 43.34590358011229,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.036594537252326705,
     0.39115195111968926
    ],
    "B": [
     0.024011565429306884,
     0.2702559115870675
    ],
    "C": [
     0.02541287309189354,
     0.2716332993886731
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00462807881773399
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "lev-350m (Franck Verrot, LFM2.5-350M)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 4.690739555547254,
    "I_sealed": 4.964385178633581,
    "base": 4.827562367090417,
    "excess": -2.8470332655245705,
    "gap": -0.2736456230863267,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 14.072218666641763,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 11.292859389745214,
       "judge": 16.941331575477918,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 16.442424242424224,
       "hard": -5.066666666666664,
       "judge": 1.4508474576270802,
       "standard": -32.55555555555556
      }
     },
     "sealed|choice": {
      "cc": 14.893155535900744,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 21.513561451481806,
       "hard": 9.898190045248851,
       "judge": 14.338919925512121,
       "standard": 22.404236974996934
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -76.0,
       "hard": -92.23300970873787,
       "judge": -81.57894736842105,
       "standard": -94.36619718309859
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 15.455319148936175,
       "hard": -14.996568627450912,
       "judge": -3.196407185628769,
       "standard": -22.131472081218284
      }
     }
    }
   },
   "jevbench_score": 0.1537555644865392,
   "key": "lev-350m",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 (code); weights under LiquidAI's LFM1.0 licence, following the LFM2.5-350M base",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6896551724137931,
    "decisive_rate": 0.07816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.86424479313416,
     "cost": 80.03797760430007,
     "intelligence": 4.827562367090417,
     "speed": 93.98230372494703
    },
    "board_release": "v1.6.43",
    "capability": 43.34590358011229,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 105,
    "capability_rank": 133,
    "composite_ci95": {
     "A": [
      0.036596960876541004,
      0.39120491517125294
     ],
     "B": [
      0.024012400167042817,
      0.27027613731690037
     ],
     "C": [
      0.02541455616426458,
      0.2716700799800367
     ]
    },
    "jevbench_score": 0.15377137469809216,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 174,
    "rank": 218,
    "ranks": {
     "A": 218,
     "B": 219,
     "C": 218,
     "capability": 194
    },
    "scores": {
     "A": 0.15377137469809216,
     "B": 0.1036628826651748,
     "C": 0.10678567687367513
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1946538500036695,
     "p50_s_raw": 0.022326925001834752,
     "p95_s_adjusted": 0.2053556374099571,
     "p95_s_raw": 0.02767781870497856
    },
    "views": {
     "Balanced 33:33:33": 0.12144625080361596,
     "Emphasis on Accuracy 60:20:20": 0.07231333345467754,
     "Emphasis on Cost 20:20:60": 0.18259637420585254,
     "Emphasis on Speed 20:60:20": 0.1852875240477895,
     "Intelligence only": 0.04500322848109108
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 240,
   "ranked": true,
   "ranks": {
    "A": 240,
    "B": 241,
    "C": 240,
    "capability": 215
   },
   "repo": "https://github.com/franckverrot/lev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.1537555644865392,
    "B": 0.10365713430715648,
    "C": 0.1067746975600967
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19497151399846188,
    "p50_s_raw": 0.022485756999230944,
    "p95_s_adjusted": 0.21566075850132618,
    "p95_s_raw": 0.032830379250663094
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "LiquidAI/LFM2.5-350M with a LoRA and a 6.5M-parameter pointer head (a kev clone)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.71542881918619,
      83.56232508492258
     ],
     "I": [
      2.9345502485864183,
      6.710083033760257
     ],
     "capability": [
      40.03125074179408,
      44.53569592988895
     ],
     "composite": {
      "A": [
       0.036594537252326705,
       0.39115195111968926
      ],
      "B": [
       0.024011565429306884,
       0.2702559115870675
      ],
      "C": [
       0.02541287309189354,
       0.2716332993886731
      ]
     },
     "seed": 60
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.86424479313416,
     "capability": 43.34590358011229,
     "intelligence": 4.827562367090417
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3333333333333333,
      "calibration": 91.32573289902281,
      "cc": 14.74285497759628,
      "cc_by_type": {
       "choice": 14.74285497759628
      },
      "cc_tiered": 14.482687101271253,
      "cc_tiered_open": 14.072218666641763,
      "cc_tiered_sealed": 14.893155535900744,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 14.74285497759628
     },
     "noul": {
      "accuracy": 0.05333333333333334,
      "calibration": 74.68787061994608,
      "cc": -89.33333333333333,
      "cc_by_type": {
       "noul": -89.33333333333333
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.248,
      "calibration": 79.5791308604336,
      "cc": -8.448333333333302,
      "cc_by_type": {
       "score": -8.448333333333302
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 81.0219883791457,
     "capability": 43.49027409042074,
     "intelligence": 5.958559801695786
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.12143310184738997,
    "Emphasis on Accuracy 60:20:20": 0.07231053614020283,
    "Emphasis on Cost 20:20:60": 0.1825785396126524,
    "Emphasis on Speed 20:60:20": 0.18523244254058632,
    "Intelligence only": 0.04500322848109108
   }
  },
  {
   "adapter_id": "images/raw-qwen3-1.7b/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 11.516326812547591,
    "cost": 68.54983190559568,
    "intelligence": 5.124286554530066,
    "speed": 93.09539316451654
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.7144612706448337,
      "ece_score": 0.0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0.0,
      "score_before_time_limit": 0.0,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0026954177897574594,
      "brier": 0.5809971640519896,
      "ece": 0.5815255292937018,
      "n": 371,
      "score": 0.0,
      "score_before_time_limit": 0.0,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.4959349593495935,
      "interval90_width": 0.23238482384823847,
      "n": 369,
      "nrps": 0.3090203912471445,
      "score": 34.548980437642776,
      "score_before_time_limit": 34.548980437642776,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.5704149904947305
     }
    },
    "score": 11.516326812547591
   },
   "capability": 8.320306683538828,
   "class": "raw-logit-control",
   "composite_ci95": {
    "A": [
     0.0406811850828886,
     0.3224784339880365
    ],
    "B": [
     0.029351742379160807,
     0.25963128860191803
    ],
    "C": [
     0.028250822974228193,
     0.2239433569361365
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011177352216748768
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 1.7B direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 7.018912428422589,
    "I_sealed": 3.2296606806375423,
    "base": 5.124286554530066,
    "excess": 1.2158641053468031,
    "gap": 3.789251747785047,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 21.05673728526777,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 41.621621621621614,
       "hard": -0.03145643284050508,
       "judge": 24.851680949241928,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 16.019988364571248,
       "hard": -4.886927335935498,
       "judge": -32.17506026172301,
       "standard": -190.1002337898353
      }
     },
     "sealed|choice": {
      "cc": 9.688982041912627,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 21.513561451481806,
       "hard": -5.3733031674208345,
       "judge": 21.787709497206727,
       "standard": 15.753171572853812
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -24.0,
       "hard": -22.330097087378643,
       "judge": -5.263157894736848,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -19.19653010919571,
       "hard": -25.671423213473844,
       "judge": -56.434399118166525,
       "standard": -31.617468115507542
      }
     }
    }
   },
   "jevbench_score": 0.13671222087234564,
   "key": "raw-qwen3-1.7b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.4189189189189189,
    "decisive_rate": 0.9973045822102425,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 11.516326812547591,
     "cost": 68.54983190559568,
     "intelligence": 5.124286554530066,
     "speed": 93.23530725162705
    },
    "board_release": "v1.6.43",
    "capability": 8.320306683538828,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 139,
    "capability_rank": 169,
    "composite_ci95": {
     "A": [
      0.040682757177666354,
      0.3224989584923113
     ],
     "B": [
      0.029352394526125654,
      0.25964193163917776
     ],
     "C": [
      0.02825191470671274,
      0.22395761006410508
     ]
    },
    "jevbench_score": 0.13671939226506033,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 175,
    "rank": 219,
    "ranks": {
     "A": 219,
     "B": 218,
     "C": 219,
     "capability": 241
    },
    "scores": {
     "A": 0.13671939226506033,
     "B": 0.10452214659357986,
     "C": 0.09494402240629189
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21396372243762016,
     "p50_s_raw": 0.03198186121881008,
     "p95_s_adjusted": 0.221885605994612,
     "p95_s_raw": 0.03594280299730599
    },
    "views": {
     "Balanced 33:33:33": 0.14292662698061412,
     "Emphasis on Accuracy 60:20:20": 0.08598558171809113,
     "Emphasis on Cost 20:20:60": 0.21037073826496294,
     "Emphasis on Speed 20:60:20": 0.21708826744822254,
     "Intelligence only": 0.05382204747082655
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 241,
   "ranked": true,
   "ranks": {
    "A": 241,
    "B": 240,
    "C": 241,
    "capability": 263
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-1.7B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.13671222087234564,
    "B": 0.10451879339909785,
    "C": 0.09493904227246223
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2148456271737814,
    "p50_s_raw": 0.0324228135868907,
    "p95_s_adjusted": 0.22820973400957878,
    "p95_s_raw": 0.0391048670047894
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-1.7B BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      11.044864988748188,
      11.991879282296361
     ],
     "I": [
      3.263077550449943,
      7.14487317844146
     ],
     "capability": [
      7.338168404696409,
      9.339947930556471
     ],
     "composite": {
      "A": [
       0.0406811850828886,
       0.3224784339880365
      ],
      "B": [
       0.029351742379160807,
       0.25963128860191803
      ],
      "C": [
       0.028250822974228193,
       0.2239433569361365
      ]
     },
     "seed": 95
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 11.516326812547591,
     "capability": 8.320306683538828,
     "intelligence": 5.124286554530066
    },
    "per_type": {
     "choice": {
      "accuracy": 0.30666666666666664,
      "calibration": 0.0,
      "cc": 11.332569176700131,
      "cc_by_type": {
       "choice": 11.332569176700131
      },
      "cc_tiered": 15.372859663590198,
      "cc_tiered_open": 21.05673728526777,
      "cc_tiered_sealed": 9.688982041912627,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 11.332569176700131
     },
     "noul": {
      "accuracy": 0.41333333333333333,
      "calibration": 0.0,
      "cc": -17.333333333333336,
      "cc_by_type": {
       "noul": -17.333333333333336
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.304,
      "calibration": 34.548980437642776,
      "cc": -31.4596497298246,
      "cc_by_type": {
       "score": -31.4596497298246
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 11.45520847967844,
     "capability": 9.226470190380851,
     "intelligence": 6.997731901083261
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.14291617738893025,
    "Emphasis on Accuracy 60:20:20": 0.08598331239756947,
    "Emphasis on Cost 20:20:60": 0.21035715517256542,
    "Emphasis on Speed 20:60:20": 0.21704488009195028,
    "Intelligence only": 0.05382204747082655
   }
  },
  {
   "adapter_id": "images/kev-0.5b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 61.20618525995795,
    "cost": 80.09398520305423,
    "intelligence": 4.598433868646836,
    "speed": 92.39061085463217
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22209098434273397,
      "ece_score": 55.5818031314532,
      "mean_tvd": null,
      "n": 729,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 55.5818031314532,
      "score_before_time_limit": 55.5818031314532,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.7439353099730458,
      "brier": 0.291543126684636,
      "ece": 0.1912938005390836,
      "n": 371,
      "score": 61.74123989218327,
      "score_before_time_limit": 61.74123989218327,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9239130434782609,
      "interval90_width": 0.835824275362319,
      "n": 368,
      "nrps": 0.21140059009020898,
      "score": 66.29551275623737,
      "score_before_time_limit": 66.29551275623737,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.23134457739252184
     }
    },
    "score": 61.20618525995795
   },
   "capability": 32.90230956430239,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.024058260427383477,
     0.35808276670744504
    ],
    "B": [
     0.015784479026651747,
     0.24966341865097394
    ],
    "C": [
     0.01670712529679409,
     0.24866858799128133
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004608226600985222
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 0.5B",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 4.849742544692461,
    "I_sealed": 4.3471251926012116,
    "base": 4.598433868646836,
    "excess": -2.0707702903469944,
    "gap": 0.5026173520912494,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 14.549227634077383,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": 11.292859389745214,
       "judge": 24.851680949241928,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -82.6086956521739,
       "judge": -100.0,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 17.000794018795805,
       "hard": -8.594117745107855,
       "judge": -32.75818090283609,
       "standard": -52.88804065591748
      }
     },
     "sealed|choice": {
      "cc": 13.041375577803636,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 17.522725593082576,
       "hard": 7.352941176470571,
       "judge": 18.06331471135942,
       "standard": 14.644660672496622
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -48.0,
       "hard": -78.64077669902912,
       "judge": -71.05263157894737,
       "standard": -66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 4.208590362582387,
       "hard": -22.033232980160843,
       "judge": -8.39299678470844,
       "standard": -23.154804312918653
      }
     }
    }
   },
   "jevbench_score": 0.1315881177953595,
   "key": "kev-0.5b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5578947368421052,
    "decisive_rate": 0.2560646900269542,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 61.20618525995795,
     "cost": 80.09398520305423,
     "intelligence": 4.598433868646836,
     "speed": 92.39904218127765
    },
    "board_release": "v1.6.43",
    "capability": 32.90230956430239,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 125,
    "capability_rank": 154,
    "composite_ci95": {
     "A": [
      0.024058315307534144,
      0.35808460603860887
     ],
     "B": [
      0.015784497925561473,
      0.2496641339569836
     ],
     "C": [
      0.016707163408009824,
      0.2486698653045895
     ]
    },
    "jevbench_score": 0.131588623265787,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 176,
    "rank": 220,
    "ranks": {
     "A": 220,
     "B": 220,
     "C": 220,
     "capability": 221
    },
    "scores": {
     "A": 0.131588623265787,
     "B": 0.08911346438551963,
     "C": 0.09138098837901876
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2379663866944611,
     "p50_s_raw": 0.043983193347230554,
     "p95_s_adjusted": 0.24186897580511868,
     "p95_s_raw": 0.04593448790255934
    },
    "views": {
     "Balanced 33:33:33": 0.10538840072414271,
     "Emphasis on Accuracy 60:20:20": 0.06258833605506466,
     "Emphasis on Cost 20:20:60": 0.15914261116628434,
     "Emphasis on Speed 20:60:20": 0.1611592968234279,
     "Intelligence only": 0.03889464633042005
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 242,
   "ranked": true,
   "ranks": {
    "A": 242,
    "B": 242,
    "C": 242,
    "capability": 243
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.1315881177953595,
    "B": 0.08911327893140526,
    "C": 0.09138063735788855
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2380974093452096,
    "p50_s_raw": 0.0440487046726048,
    "p95_s_adjusted": 0.24220563587732613,
    "p95_s_raw": 0.04610281793866307
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen2.5-0.5B + LoRA + learned pointer head; jaredpalmer/kev-0.5b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.41312884823437,
      65.28644013850979
     ],
     "I": [
      2.5515112413087477,
      6.5563351690373795
     ],
     "capability": [
      29.53128001342342,
      35.43777813183522
     ],
     "composite": {
      "A": [
       0.024058260427383477,
       0.35808276670744504
      ],
      "B": [
       0.015784479026651747,
       0.24966341865097394
      ],
      "C": [
       0.01670712529679409,
       0.24866858799128133
      ]
     },
     "seed": 52
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 61.20618525995795,
     "capability": 32.90230956430239,
     "intelligence": 4.598433868646836
    },
    "per_type": {
     "choice": {
      "accuracy": 0.31866666666666665,
      "calibration": 55.5818031314532,
      "cc": 12.8671977871034,
      "cc_by_type": {
       "choice": 12.8671977871034
      },
      "cc_tiered": 13.795301605940509,
      "cc_tiered_open": 14.549227634077383,
      "cc_tiered_sealed": 13.041375577803636,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 12.8671977871034
     },
     "noul": {
      "accuracy": 0.14133333333333334,
      "calibration": 61.74123989218327,
      "cc": -71.73333333333333,
      "cc_by_type": {
       "noul": -71.73333333333333
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.272,
      "calibration": 66.29551275623737,
      "cc": -15.865897379211603,
      "cc_by_type": {
       "score": -15.865897379211603
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 65.8831405050141,
     "capability": 35.92446865234692,
     "intelligence": 5.965796799679742
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.028000000000000025,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.10538796842605185,
    "Emphasis on Accuracy 60:20:20": 0.06258824457280192,
    "Emphasis on Cost 20:20:60": 0.15914201971079578,
    "Emphasis on Speed 20:60:20": 0.16115747721558535,
    "Intelligence only": 0.03889464633042005
   }
  },
  {
   "adapter_id": "images/openjev-verdict/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Hemant (heman10x)",
   "axes": {
    "calibration": 49.11476555745841,
    "cost": 86.62313527343974,
    "intelligence": 4.1496399816416485,
    "speed": 78.45185691662326
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.3268574231249395,
      "ece_score": 34.6285153750121,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 34.6285153750121,
      "score_before_time_limit": 34.6285153750121,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.6666666666666667,
      "brier": 0.28308617598601715,
      "ece": 0.20554105202175244,
      "n": 375,
      "score": 58.89178959564951,
      "score_before_time_limit": 58.89178959564951,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.816,
      "interval90_width": 0.6600000000000001,
      "n": 375,
      "nrps": 0.21771137902642015,
      "score": 53.823991701713624,
      "score_before_time_limit": 53.823991701713624,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.3529043934696537
     }
    },
    "score": 49.11476555745841
   },
   "capability": 26.63220276955003,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.016675550357464684,
     0.2745367164429247
    ],
    "B": [
     0.010945992361712387,
     0.1924055928378838
    ],
    "C": [
     0.011580243303794922,
     0.1906504975298088
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.002791871921182266
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "openJev Verdict (heman10x, ModernBERT-base 151M)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 5.710480975731317,
    "I_sealed": 2.5887989875519795,
    "base": 4.1496399816416485,
    "excess": 0.5482943457410934,
    "gap": 3.121681988179337,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 17.13144292719395,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": 13.180245360176167,
       "judge": 20.896506262359924,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -56.52173913043479,
       "judge": -80.0,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 8.056046691320095,
       "hard": -27.078088834020786,
       "judge": -9.574770356665475,
       "standard": -54.57566450814868
      }
     },
     "sealed|choice": {
      "cc": 7.766396962655938,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 10.87133249575052,
       "hard": 5.316742081447947,
       "judge": 6.890130353817524,
       "standard": 12.427638871782255
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -64.0,
       "hard": -49.51456310679612,
       "judge": -63.1578947368421,
       "standard": -60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 11.069847893733552,
       "hard": -5.166740337464448,
       "judge": -15.492495207027046,
       "standard": -16.23629733590537
      }
     }
    }
   },
   "jevbench_score": 0.09645563919820467,
   "key": "openjev-verdict",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6,
    "decisive_rate": 0.3333333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 49.11476555745841,
     "cost": 86.62313527343974,
     "intelligence": 4.1496399816416485,
     "speed": 79.40046783928
    },
    "board_release": "v1.6.43",
    "capability": 26.63220276955003,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 131,
    "capability_rank": 161,
    "composite_ci95": {
     "A": [
      0.016680739292552642,
      0.27473431836523593
     ],
     "B": [
      0.010947780723182459,
      0.1924832136378699
     ],
     "C": [
      0.011583846730939337,
      0.19078772108696943
     ]
    },
    "jevbench_score": 0.09650709186186587,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 177,
    "rank": 221,
    "ranks": {
     "A": 221,
     "B": 221,
     "C": 221,
     "capability": 232
    },
    "scores": {
     "A": 0.09650709186186587,
     "B": 0.06541514759096348,
     "C": 0.0670188137929624
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.6885442301630974,
     "p50_s_raw": 0.2692721150815487,
     "p95_s_adjusted": 1.66732926396653,
     "p95_s_raw": 0.7586646319832651
    },
    "views": {
     "Balanced 33:33:33": 0.07793885393113759,
     "Emphasis on Accuracy 60:20:20": 0.046097376471726674,
     "Emphasis on Cost 20:20:60": 0.11949201576637343,
     "Emphasis on Speed 20:60:20": 0.11862755697857134,
     "Intelligence only": 0.028581910146043114
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 244,
   "ranked": true,
   "ranks": {
    "A": 244,
    "B": 244,
    "C": 244,
    "capability": 254
   },
   "repo": "https://github.com/Heman10x-NGU/openJev-verdict-2.0",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.09645563919820467,
    "B": 0.06539623103617571,
    "C": 0.066983082776531
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7912336127832532,
    "p50_s_raw": 0.3206168063916266,
    "p95_s_adjusted": 1.8051345552317797,
    "p95_s_raw": 0.8275672776158899
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "GLiClass ModernBERT-base (knowledgator/gliclass-modern-base-v2.0) fine-tuned, 151M",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      43.627592951745086,
      53.02973228229396
     ],
     "I": [
      2.2588266004870072,
      6.027430783705419
     ],
     "capability": [
      23.293166910490978,
      29.139107914545924
     ],
     "composite": {
      "A": [
       0.016675550357464684,
       0.2745367164429247
      ],
      "B": [
       0.010945992361712387,
       0.1924055928378838
      ],
      "C": [
       0.011580243303794922,
       0.1906504975298088
      ]
     },
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 49.11476555745841,
     "capability": 26.63220276955003,
     "intelligence": 4.1496399816416485
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2906666666666667,
      "calibration": 34.6285153750121,
      "cc": 9.286397696162448,
      "cc_by_type": {
       "choice": 9.286397696162448
      },
      "cc_tiered": 12.448919944924945,
      "cc_tiered_open": 17.13144292719395,
      "cc_tiered_sealed": 7.766396962655938,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 9.286397696162448
     },
     "noul": {
      "accuracy": 0.2,
      "calibration": 58.89178959564951,
      "cc": -60.0,
      "cc_by_type": {
       "noul": -60.0
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2693333333333333,
      "calibration": 53.823991701713624,
      "cc": -9.675453706526316,
      "cc_by_type": {
       "score": -9.675453706526316
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 49.678974901443674,
     "capability": 27.759290067036318,
     "intelligence": 5.839605232628962
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.07789411162231974,
    "Emphasis on Accuracy 60:20:20": 0.04608798194516172,
    "Emphasis on Cost 20:20:60": 0.11942891140749813,
    "Emphasis on Speed 20:60:20": 0.1184411678748766,
    "Intelligence only": 0.028581910146043114
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 60.96568104722222,
    "cost": 79.56261245432609,
    "intelligence": 4.019045994532796,
    "speed": 94.90280801143004
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22390249999999998,
      "ece_score": 55.2195,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 55.2195,
      "score_before_time_limit": 55.2195,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.712,
      "brier": 0.28332537661333335,
      "ece": 0.2195469333333333,
      "n": 375,
      "score": 56.09061333333334,
      "score_before_time_limit": 56.09061333333334,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9253333333333333,
      "interval90_width": 0.8177777777777777,
      "n": 375,
      "nrps": 0.1894891371666667,
      "score": 71.58692980833334,
      "score_before_time_limit": 71.58692980833334,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.18938613333333326
     }
    },
    "score": 60.96568104722222
   },
   "capability": 32.492363520877504,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.020959591958551866,
     0.3032092824492544
    ],
    "B": [
     0.01370001323335493,
     0.2103075700089039
    ],
    "C": [
     0.014555272193438792,
     0.21056200170087108
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra same-size encoders (bge-large, e5-large, Qwen3-Embedding-0.6B) $0.01/M input (SEA-LION-ModernBERT-300M has no market listing; the ModernBERT-large-class reference of the Laya/Certo/OpenDecision rows) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004800055418719211
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Small (Chinh Nguyen, SEA-LION-ModernBERT-300M encoder with fixed heads)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 4.071843176603614,
    "I_sealed": 3.966248812461978,
    "base": 4.019045994532796,
    "excess": -2.467793278296608,
    "gap": 0.10559436414163592,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 12.215529529810842,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.702702702702708,
       "hard": 5.630701478452354,
       "judge": 20.896506262359924,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -56.52173913043479,
       "judge": -40.0,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 13.416060606060597,
       "hard": -26.448333333333316,
       "judge": -3.2088135593220546,
       "standard": -76.01518518518517
      }
     },
     "sealed|choice": {
      "cc": 11.898746437385935,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 17.522725593082576,
       "hard": 5.316742081447947,
       "judge": 19.92551210428307,
       "standard": 10.210617071067881
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -72.0,
       "hard": -59.22330097087378,
       "judge": -65.78947368421053,
       "standard": -46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 20.691063829787236,
       "hard": -7.6461764705882285,
       "judge": -0.1282634730539245,
       "standard": 0.04786802030454229
      }
     }
    }
   },
   "jevbench_score": 0.08963662915567427,
   "key": "quyet-1-0-small",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7129629629629629,
    "decisive_rate": 0.288,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 60.96568104722222,
     "cost": 79.56261245432609,
     "intelligence": 4.019045994532796,
     "speed": 94.91846454857637
    },
    "board_release": "v1.6.43",
    "capability": 32.492363520877504,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 127,
    "capability_rank": 156,
    "composite_ci95": {
     "A": [
      0.020959672741331675,
      0.3032118958147926
     ],
     "B": [
      0.01370004084448916,
      0.2103085758125444
     ],
     "C": [
      0.014555328292591438,
      0.21056381653805042
     ]
    },
    "jevbench_score": 0.08963716950381444,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 178,
    "rank": 222,
    "ranks": {
     "A": 222,
     "B": 222,
     "C": 222,
     "capability": 224
    },
    "scores": {
     "A": 0.08963716950381444,
     "B": 0.0601437708059232,
     "C": 0.06224803437764891
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.17903870801674202,
     "p50_s_raw": 0.01451935400837101,
     "p95_s_adjusted": 0.17997269080078693,
     "p95_s_raw": 0.014986345400393473
    },
    "views": {
     "Balanced 33:33:33": 0.07128318534334778,
     "Emphasis on Accuracy 60:20:20": 0.04197968509922768,
     "Emphasis on Cost 20:20:60": 0.10875180747478416,
     "Emphasis on Speed 20:60:20": 0.11026128891416027,
     "Intelligence only": 0.025967427058159954
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 245,
   "ranked": true,
   "ranks": {
    "A": 245,
    "B": 245,
    "C": 245,
    "capability": 246
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.08963662915567427,
    "B": 0.06014357619379634,
    "C": 0.06224765913588491
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.17911875999416224,
    "p50_s_raw": 0.014559379997081123,
    "p95_s_adjusted": 0.18054194830911,
    "p95_s_raw": 0.015270974154555006
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "aisingapore/SEA-LION-ModernBERT-300M",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.688303752892594,
      64.99464873956296
     ],
     "I": [
      2.4305093708220933,
      6.1818340512057315
     ],
     "capability": [
      29.486356353832495,
      35.15105810056002
     ],
     "composite": {
      "A": [
       0.020959591958551866,
       0.3032092824492544
      ],
      "B": [
       0.01370001323335493,
       0.2103075700089039
      ],
      "C": [
       0.014555272193438792,
       0.21056200170087108
      ]
     },
     "seed": 88
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 60.96568104722222,
     "capability": 32.492363520877504,
     "intelligence": 4.019045994532796
    },
    "per_type": {
     "choice": {
      "accuracy": 0.30533333333333335,
      "calibration": 55.2195,
      "cc": 11.162054886655328,
      "cc_by_type": {
       "choice": 11.162054886655328
      },
      "cc_tiered": 12.057137983598388,
      "cc_tiered_open": 12.215529529810842,
      "cc_tiered_sealed": 11.898746437385935,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 11.162054886655328
     },
     "noul": {
      "accuracy": 0.20533333333333334,
      "calibration": 56.09061333333334,
      "cc": -58.93333333333332,
      "cc_by_type": {
       "noul": -58.93333333333332
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2986666666666667,
      "calibration": 71.58692980833334,
      "cc": -4.271587719298253,
      "cc_by_type": {
       "score": -4.271587719298253
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 55.62018953114671,
     "capability": 29.685347509548446,
     "intelligence": 3.750505487950187
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.07128272971559697,
    "Emphasis on Accuracy 60:20:20": 0.041979590286340934,
    "Emphasis on Cost 20:20:60": 0.10875117117698081,
    "Emphasis on Speed 20:60:20": 0.11025932668532612,
    "Intelligence only": 0.025967427058159954
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zaitgeist Labs / Christian Oezbek (c.oezbek@gmail.com, GitHub @coezbek / @zaitgeist-charly)",
   "axes": {
    "calibration": 79.37888213235459,
    "cost": 89.00253166733508,
    "intelligence": 3.759262959387639,
    "speed": 73.82860858123392
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13684690553745932,
      "ece_score": 72.63061889250814,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 72.63061889250814,
      "score_before_time_limit": 72.63061889250814,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.9865229110512129,
      "brier": 0.2509324734501347,
      "ece": 0.08358625336927226,
      "n": 371,
      "score": 83.28274932614555,
      "score_before_time_limit": 83.28274932614555,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967479674796748,
      "interval90_width": 0.8308491418247516,
      "n": 369,
      "nrps": 0.16220543914182478,
      "score": 82.22327817841011,
      "score_before_time_limit": 82.22327817841011,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09666449864498641
     }
    },
    "score": 79.37888213235459
   },
   "capability": 41.569072545871116,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.004014997550795856,
     0.37828572053602627
    ],
    "B": [
     0.002573281180911842,
     0.2630825532407063
    ],
    "C": [
     0.0027881927436082337,
     0.2626984170389072
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no public tariff). THE SAME CHOICE THE PUBLISHED quyet-1-0-tiny ROW AND RUN 59'S tacet-sonata ROW CARRY FOR THE IDENTICAL mmBERT-small BACKBONE, carried unchanged and not a new judgement: DeepInfra base-size encoders at USD 0.005 per 1M input, 0 output - deepinfra:thenlper/gte-base, deepinfra:BAAI/bge-base-en-v1.5, deepinfra:intfloat/e5-base-v2 and deepinfra:sentence-transformers/all-mpnet-base-v2 are all (0.005, 0.0) in the frozen 25 Sep snapshot - x this row's own measured input tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, verified in the author's source: watt_flash/serve.py:answer() formats a softmax over the option markers of one encoder pass and the package contains no generation path at all (the 02 Oct deputy rule for classifier read-outs). THE SERVER REPORTS NO usage BLOCK OF ANY KIND, so the input counts come EXACTLY from the author's own unmodified watt_flash.layout.Layout.pack_request - the very number their own WattFlash.decide_batch returns as len(p) and only serve.py drops - under PREREG-R61-AMENDMENT-1, cross-checked three ways (independent reassembly of their documented layout; identity of their packer's Overflow set with the server's own 422 set), with no cost number at all on any disagreement; the counts sit in cost_estimate and never in usage. RELEASE-LANE LINE, recorded and not acted on, AND IT IS THE CARRIED quyet/tacet LINE, NOT A NEW ONE: jhu-clsp/mmBERT-small still has no explicit market-listing decision (pricing_v15.reference() raises), so an UNLISTED_BASE_MODELS entry is the release lane's.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.00229019
    },
    "kind": "mixed",
    "usage_estimated_rows": 1477,
    "usd_per_1000": 0.0023258530805687202
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "watt-flash-0.1 (Zaitgeist Labs, 140M mmBERT-small encoder with an option-marker scorer, one forward pass per request, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights zaitlabs/watt-flash-0.1 @ d49a3a880b0201b325f2d2299d4c36ffc8735477 (Apache-2.0, public and ungated), model.safetensors sha256 099e611bd27c9542732b89be47bcace54964c00d9c7161ebccb061648c9780bf = THE SUBMITTER'S OWN DECLARED HASH, loaded from a local snapshot with the author's own --model flag (no network during the run). Code github.com/zaitgeist-charly/zaitlabs-watt-flash-0.1 @ e9863f06230ad874a5a2cfca3af922e45161a4d7 (Apache-2.0, = HEAD of main), installed with the author's own line `uv sync` against their own uv.lock in an isolated venv (torch 2.14.0, transformers 5.17.0, safetensors 0.8.0) and served with their own documented command `python -m watt_flash.serve --port 8942`. No file of theirs was edited. Their server answers POST /v1/systemone with Jev's shapes (noul -> {noul: P(true)}; choice probabilities keyed by the criteria keys with choice = argmax; score probabilities keyed '0'..'n-1'), so the UNCHANGED typesafe adapter applies and no mapping is ours; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE AUTHOR'S FREE HOSTED ENDPOINT api.inference.zaitlabs.com WAS NEVER USED - the submission states it may train on the data, so no sealed item reached it. THE LONG ITEMS ARE REFUSED, NOT TRUNCATED, by the author's own code: watt_flash/layout.py raises Overflow past its documented 8,192-token window and serve.py answers that with HTTP 422, which their own commit e9863f06 says is deliberate ('answer an over-long request with 422, which JevBench scores as a refusal, not an outage'); those items are counted wrong and no other item failed. A RESUBMISSION of GitHub #113 via the form, which the author asks to dedupe with it. RUN ON SANDY'S SHARED CPU (12-core Zen 2, AVX2, fp32 - the package itself calls model.float() off-accelerator, 8 threads, nice), server and driver inside a loopback-only network namespace (unshare -rn) with no credential in the environment; load average logged every 60 s, so the Speed axis is a DECLARED LOWER BOUND. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline. add-requests run 61, benchmarkheaven.com/submit ref 0874B655 (bh-submit 4cf0bdfb-b6d7-4777-a8ac-350d29c5ac3f)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "CPU (12-core Zen 2, AVX2; 8 threads)",
   "intelligence": {
    "I_open": 5.1066210946590145,
    "I_sealed": 2.4119048241162644,
    "base": 3.759262959387639,
    "excess": 0.12132862810450629,
    "gap": 2.69471627054275,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 9.15774230012725,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 26.05405405405406,
       "hard": -1.9188424032714584,
       "judge": 12.986156888595913,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 6.162120983849794,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 20.743030303030295,
       "hard": -4.7528703703703545,
       "judge": 12.24322033898303,
       "standard": 11.579999999999991
      }
     },
     "sealed|choice": {
      "cc": 1.205164808546094,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 38.80718350454514,
       "hard": -2.3190045248868976,
       "judge": -8.9385474860335,
       "standard": 4.668062569281944
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -92.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "sealed|score": {
      "cc": 6.0305496638027005,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 12.903262411347516,
       "hard": -5.752573529411742,
       "judge": 16.985089820359278,
       "standard": 9.728629441624314
      }
     }
    }
   },
   "jevbench_score": 0.07452931157363736,
   "key": "watt-flash-0.1",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "zaitlabs/watt-flash-0.1@d49a3a880b0201b325f2d2299d4c36ffc8735477",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6,
    "decisive_rate": 0.013477088948787063,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.37888213235459,
     "cost": 89.00253166733508,
     "intelligence": 3.759262959387639,
     "speed": 75.84832436427908
    },
    "board_release": "v1.6.43",
    "capability": 41.569072545871116,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 111,
    "capability_rank": 139,
    "composite_ci95": {
     "A": [
      0.004016900605731708,
      0.3790125753277212
     ],
     "B": [
      0.0025739064184890927,
      0.26336355668719996
     ],
     "C": [
      0.0027895143095359085,
      0.2632031773109175
     ]
    },
    "jevbench_score": 0.07461802002995101,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 179,
    "rank": 223,
    "ranks": {
     "A": 223,
     "B": 223,
     "C": 223,
     "capability": 203
    },
    "scores": {
     "A": 0.07461802002995101,
     "B": 0.049670099364146944,
     "C": 0.05181806946524376
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.0144243478775024,
     "p50_s_raw": 0.4322121739387512,
     "p95_s_adjusted": 2.564176402240991,
     "p95_s_raw": 1.2070882011204955
    },
    "views": {
     "Balanced 33:33:33": 0.05839101580851302,
     "Emphasis on Accuracy 60:20:20": 0.03436581390194284,
     "Emphasis on Cost 20:20:60": 0.09032934623751702,
     "Emphasis on Speed 20:60:20": 0.08921814434421181,
     "Intelligence only": 0.02125044886845491
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 246,
   "ranked": true,
   "ranks": {
    "A": 246,
    "B": 246,
    "C": 246,
    "capability": 226
   },
   "repo": "https://huggingface.co/zaitlabs/watt-flash-0.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.07452931157363736,
    "B": 0.04963863644240823,
    "C": 0.051756466370581504
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.1312192991375922,
    "p50_s_raw": 0.49060964956879616,
    "p95_s_adjusted": 3.660937681049108,
    "p95_s_raw": 1.755468840524554
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "jhu-clsp/mmBERT-small",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      74.23510907541693,
      82.71222317398025
     ],
     "I": [
      1.3821722420456268,
      6.668265912418397
     ],
     "capability": [
      38.3327917105868,
      43.940510516442146
     ],
     "composite": {
      "A": [
       0.004014997550795856,
       0.37828572053602627
      ],
      "B": [
       0.002573281180911842,
       0.2630825532407063
      ],
      "C": [
       0.0027881927436082337,
       0.2626984170389072
      ]
     },
     "seed": 150
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.37888213235459,
     "capability": 41.569072545871116,
     "intelligence": 3.759262959387639
    },
    "per_type": {
     "choice": {
      "accuracy": 0.25866666666666666,
      "calibration": 72.63061889250814,
      "cc": 5.194054735087065,
      "cc_by_type": {
       "choice": 5.194054735087065
      },
      "cc_tiered": 5.181453554336672,
      "cc_tiered_open": 9.15774230012725,
      "cc_tiered_sealed": 1.205164808546094,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 5.194054735087065
     },
     "noul": {
      "accuracy": 0.008,
      "calibration": 83.28274932614555,
      "cc": -98.4,
      "cc_by_type": {
       "noul": -98.4
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.368,
      "calibration": 82.22327817841011,
      "cc": 4.764543859649162,
      "cc_by_type": {
       "score": 4.764543859649162
      },
      "cc_tiered": 6.096335323826247,
      "cc_tiered_open": 6.162120983849794,
      "cc_tiered_sealed": 6.0305496638027005,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 4.764543859649162
     }
    },
    "raw_on_A": {
     "calibration": 80.11428356201952,
     "capability": 42.17074394730905,
     "intelligence": 4.227204332598573
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.05831859122894133,
    "Emphasis on Accuracy 60:20:20": 0.03435074966868935,
    "Emphasis on Cost 20:20:60": 0.09022534426099546,
    "Emphasis on Speed 20:60:20": 0.08891445399924461,
    "Intelligence only": 0.02125044886845491
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 58.46458026762962,
    "cost": 88.43590631389968,
    "intelligence": 3.6964771541368573,
    "speed": 95.07796177344144
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2315684375,
      "ece_score": 53.6863125,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 53.6863125,
      "score_before_time_limit": 53.6863125,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.8026666666666666,
      "brier": 0.28768576031999976,
      "ece": 0.19567466666666677,
      "n": 375,
      "score": 60.86506666666664,
      "score_before_time_limit": 60.86506666666664,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8933333333333333,
      "interval90_width": 0.7453333333333331,
      "n": 375,
      "nrps": 0.1976125006088888,
      "score": 60.84236163622223,
      "score_before_time_limit": 60.84236163622223,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.2927701333333333
     }
    },
    "score": 58.46458026762962
   },
   "capability": 31.08052871088324,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.019766201285679498,
     0.27340010410511306
    ],
    "B": [
     0.012911538304013407,
     0.1889551228714117
    ],
    "C": [
     0.013726528670610767,
     0.18986118340632852
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (mmBERT-small 16-layer, 183M; base-size class reference, errs high vs the small-encoder $0.004 class) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0024292364532019703
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Tiny (Chinh Nguyen, mmBERT-small encoder (16 layers kept) with fixed heads)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 1.2549826648038989,
    "I_sealed": 6.1379716434698155,
    "base": 3.6964771541368573,
    "excess": -7.456376621104161,
    "gap": -4.882988978665916,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 3.764947994411697,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -12.86486486486486,
       "hard": 5.630701478452354,
       "judge": 12.986156888595913,
       "standard": -5.4834658853076625
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -56.52173913043479,
       "judge": -80.0,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -8.307575757575748,
       "hard": -21.650555555555574,
       "judge": -15.678474576271228,
       "standard": -51.14888888888891
      }
     },
     "sealed|choice": {
      "cc": 16.468453763405932,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 12.201611115216933,
       "hard": 14.479638009049756,
       "judge": 18.06331471135942,
       "standard": 20.18721517428256
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -84.0,
       "hard": -68.93203883495146,
       "judge": -71.05263157894737,
       "standard": -83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 1.9454611670035158,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 16.69851063829787,
       "hard": -2.889509803921597,
       "judge": -1.9068263473054348,
       "standard": 10.017309644669991
      }
     }
    }
   },
   "jevbench_score": 0.07064720497480408,
   "key": "quyet-1-0-tiny",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6351351351351351,
    "decisive_rate": 0.19733333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 58.46458026762962,
     "cost": 88.43590631389968,
     "intelligence": 3.6964771541368573,
     "speed": 95.09610910731021
    },
    "board_release": "v1.6.43",
    "capability": 31.08052871088324,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 128,
    "capability_rank": 157,
    "composite_ci95": {
     "A": [
      0.019766287637893492,
      0.2734027471253872
     ],
     "B": [
      0.012911567780248491,
      0.18895613284128507
     ],
     "C": [
      0.013726588637426037,
      0.1898630188370744
     ]
    },
    "jevbench_score": 0.07064766318734213,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 180,
    "rank": 224,
    "ranks": {
     "A": 224,
     "B": 224,
     "C": 224,
     "capability": 226
    },
    "scores": {
     "A": 0.07064766318734213,
     "B": 0.04711839739648976,
     "C": 0.04906087721343202
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1753336349996971,
     "p50_s_raw": 0.012666817499848548,
     "p95_s_adjusted": 0.17641026510682423,
     "p95_s_raw": 0.013205132553412113
    },
    "views": {
     "Balanced 33:33:33": 0.05608574548219273,
     "Emphasis on Accuracy 60:20:20": 0.03279057174059279,
     "Emphasis on Cost 20:20:60": 0.08676445199059092,
     "Emphasis on Speed 20:60:20": 0.08720297577636826,
     "Intelligence only": 0.020203381773039073
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 247,
   "ranked": true,
   "ranks": {
    "A": 247,
    "B": 247,
    "C": 247,
    "capability": 247
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Tiny",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.07064720497480408,
    "B": 0.0471182343378976,
    "C": 0.0490605590102806
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1755638730013743,
    "p50_s_raw": 0.012781936500687152,
    "p95_s_adjusted": 0.17691663471632635,
    "p95_s_raw": 0.013458317358163184
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "jhu-clsp/mmBERT-small",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      52.6105536767426,
      62.31255872829814
     ],
     "I": [
      2.382388771227772,
      5.956289831309647
     ],
     "capability": [
      27.937688273976466,
      33.75728623357733
     ],
     "composite": {
      "A": [
       0.019766201285679498,
       0.27340010410511306
      ],
      "B": [
       0.012911538304013407,
       0.1889551228714117
      ],
      "C": [
       0.013726528670610767,
       0.18986118340632852
      ]
     },
     "seed": 90
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 58.46458026762962,
     "capability": 31.08052871088324,
     "intelligence": 3.6964771541368573
    },
    "per_type": {
     "choice": {
      "accuracy": 0.32133333333333336,
      "calibration": 53.6863125,
      "cc": 13.20822636719302,
      "cc_by_type": {
       "choice": 13.20822636719302
      },
      "cc_tiered": 10.116700878908814,
      "cc_tiered_open": 3.764947994411697,
      "cc_tiered_sealed": 16.468453763405932,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.20822636719302
     },
     "noul": {
      "accuracy": 0.12533333333333332,
      "calibration": 60.86506666666664,
      "cc": -74.93333333333334,
      "cc_by_type": {
       "noul": -74.93333333333334
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.33866666666666667,
      "calibration": 60.84236163622223,
      "cc": -1.831508771929835,
      "cc_by_type": {
       "score": -1.831508771929835
      },
      "cc_tiered": 0.9727305835017579,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 1.9454611670035158,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 54.460514049074064,
     "capability": 29.223962167243283,
     "intelligence": 3.987410285412503
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.05608536043413992,
    "Emphasis on Accuracy 60:20:20": 0.03279049277078114,
    "Emphasis on Cost 20:20:60": 0.08676389909254054,
    "Emphasis on Speed 20:60:20": 0.08720130029466011,
    "Intelligence only": 0.020203381773039073
   }
  },
  {
   "adapter_id": "images/open-jev-deberta-v3-large/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Kotoba Labs",
   "axes": {
    "calibration": 67.96597988813339,
    "cost": 77.5922354035203,
    "intelligence": 2.8293414417857914,
    "speed": 67.31985953521365
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20927949454905329,
      "ece_score": 58.14410109018935,
      "mean_tvd": null,
      "n": 682,
      "n_ece_hard": 276,
      "n_tvd": 0,
      "score": 58.14410109018935,
      "score_before_time_limit": 58.14410109018935,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.848,
      "brier": 0.2599137925955484,
      "ece": 0.20243010437488554,
      "n": 375,
      "score": 59.5139791250229,
      "score_before_time_limit": 59.5139791250229,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8945945945945946,
      "interval90_width": 0.7720720720720718,
      "n": 370,
      "nrps": 0.16620611847475783,
      "score": 86.23985944918795,
      "score_before_time_limit": 86.23985944918795,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05449834627074163
     }
    },
    "score": 67.96597988813339
   },
   "capability": 35.39766066495959,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0016093461644738365,
     0.1644948890001326
    ],
    "B": [
     0.0010275646952196899,
     0.1126105605200805
    ],
    "C": [
     0.0011176015031068307,
     0.11423256180564761
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005583743842364532
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "open-jev-deberta-v3-large (local CPU)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 3.3193692126143426,
    "I_sealed": 2.3393136709572406,
    "base": 2.8293414417857914,
    "excess": -1.5933321007811418,
    "gap": 0.980055541657102,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 9.958107637843028,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 22.16216216216216,
       "hard": 1.855929537590448,
       "judge": 16.941331575477918,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -73.91304347826086,
       "judge": -70.0,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -3.536711678360427,
       "hard": -10.998114736543751,
       "judge": 10.982108090893682,
       "standard": -6.176287708459061
      }
     },
     "sealed|choice": {
      "cc": 5.620402697655405,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 17.522725593082576,
       "hard": -1.3009049773755852,
       "judge": 7.8212290502793484,
       "standard": 10.210617071067881
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -68.0,
       "hard": -88.3495145631068,
       "judge": -84.21052631578947,
       "standard": -91.54929577464789
      }
     },
     "sealed|score": {
      "cc": 1.3975383152163166,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -2.206062525510788,
       "hard": -5.817775736909847,
       "judge": 9.848256453782477,
       "standard": 4.953889631982955
      }
     }
    }
   },
   "jevbench_score": 0.03235273404518239,
   "key": "open-jev-deberta-v3-large",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (model card); DeBERTa-v3 keeps its own terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5789473684210527,
    "decisive_rate": 0.152,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 67.96597988813339,
     "cost": 77.5922354035203,
     "intelligence": 2.8293414417857914,
     "speed": 67.62138193768119
    },
    "board_release": "v1.6.43",
    "capability": 35.39766066495959,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.0016094499426836406,
      0.16453983269325992
     ],
     "B": [
      0.0010275985408014102,
      0.11262740891592737
     ],
     "C": [
      0.0011176735713080835,
      0.11426377270365268
     ]
    },
    "jevbench_score": 0.03235814777127837,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 184,
    "rank": 228,
    "ranks": {
     "A": 228,
     "B": 228,
     "C": 228,
     "capability": 215
    },
    "scores": {
     "A": 0.03235814777127837,
     "B": 0.021367993730971636,
     "C": 0.02247093595227665
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 3.5280169868841766,
     "p50_s_raw": 1.6890084934420884,
     "p95_s_adjusted": 4.901524059008808,
     "p95_s_raw": 2.3757620295044037
    },
    "views": {
     "Balanced 33:33:33": 0.0252055189654814,
     "Emphasis on Accuracy 60:20:20": 0.014715479135725748,
     "Emphasis on Cost 20:20:60": 0.0393479951952157,
     "Emphasis on Speed 20:60:20": 0.038983856596301915,
     "Intelligence only": 0.00905974708046916
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 251,
   "ranked": true,
   "ranks": {
    "A": 251,
    "B": 251,
    "C": 251,
    "capability": 238
   },
   "repo": "https://github.com/kotoba-lang/typed-decisions",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.03235273404518239,
    "B": 0.021366104953780864,
    "C": 0.02246717642026555
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 3.520035414583981,
    "p50_s_raw": 1.6850177072919905,
    "p95_s_adjusted": 5.265832172799856,
    "p95_s_raw": 2.5579160863999277
   },
   "status": {
    "answered_ok": 1427,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "microsoft/deberta-v3-large",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.83136705170831,
      72.03960745264857
     ],
     "I": [
      1.0164743841847181,
      4.99302167761034
     ],
     "capability": [
      32.41699387669788,
      37.92156574534849
     ],
     "composite": {
      "A": [
       0.0016093461644738365,
       0.1644948890001326
      ],
      "B": [
       0.0010275646952196899,
       0.1126105605200805
      ],
      "C": [
       0.0011176015031068307,
       0.11423256180564761
      ]
     },
     "seed": 75
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 67.96597988813339,
     "capability": 35.39766066495959,
     "intelligence": 2.8293414417857914
    },
    "per_type": {
     "choice": {
      "accuracy": 0.272,
      "calibration": 58.14410109018935,
      "cc": 6.899197635535144,
      "cc_by_type": {
       "choice": 6.899197635535144
      },
      "cc_tiered": 7.789255167749216,
      "cc_tiered_open": 9.958107637843028,
      "cc_tiered_sealed": 5.620402697655405,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 6.899197635535144
     },
     "noul": {
      "accuracy": 0.088,
      "calibration": 59.5139791250229,
      "cc": -82.4,
      "cc_by_type": {
       "noul": -82.4
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.31733333333333336,
      "calibration": 86.23985944918795,
      "cc": -0.7784697361159942,
      "cc_by_type": {
       "score": -0.7784697361159942
      },
      "cc_tiered": 0.6987691576081583,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 1.3975383152163166,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 68.63668964823074,
     "capability": 36.14242333704565,
     "intelligence": 3.64815702586055
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.09066666666666667,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.02520113914621073,
    "Emphasis on Accuracy 60:20:20": 0.01471458332965649,
    "Emphasis on Cost 20:20:60": 0.03934159097653584,
    "Emphasis on Speed 20:60:20": 0.038965003946316364,
    "Intelligence only": 0.00905974708046916
   }
  },
  {
   "adapter_id": "images/verdict-small/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Manavarya09 (Manav)",
   "axes": {
    "calibration": 72.92000413192591,
    "cost": 100,
    "intelligence": 2.3417095945172277,
    "speed": 88.74744771425449
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13790718750000003,
      "ece_score": 72.4185625,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 72.4185625,
      "score_before_time_limit": 72.4185625,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.5546666666666666,
      "brier": 0.28117829917333326,
      "ece": 0.18824453333333335,
      "n": 375,
      "score": 62.35109333333333,
      "score_before_time_limit": 62.35109333333333,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.9684444444444443,
      "n": 375,
      "nrps": 0.18136406875111122,
      "score": 83.99035656244443,
      "score_before_time_limit": 83.99035656244443,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06941440000000007
     }
    },
    "score": 72.92000413192591
   },
   "capability": 37.63085686322157,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0003475070464105525,
     0.12852579345393955
    ],
    "B": [
     0.00021949435923644138,
     0.08621537426131828
    ],
    "C": [
     0.0002413243377851058,
     0.08925402323190244
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.000866152709359606
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "verdict-small (Manavarya09, multilingual-e5-small 118M)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 4.005009650747796,
    "I_sealed": 0.6784095382866591,
    "base": 2.3417095945172277,
    "excess": 0.7532124700228935,
    "gap": 3.3266001124611373,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 12.01502895224339,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -8.972972972972968,
       "hard": 13.180245360176167,
       "judge": 9.03098220171391,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -39.13043478260869,
       "judge": -40.0,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 6.291818181818199,
       "hard": -6.231481481481493,
       "judge": 7.341694915254216,
       "standard": -11.9677777777778
      }
     },
     "sealed|choice": {
      "cc": 2.035228614859977,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -7.75256817677922,
       "hard": -4.355203619909522,
       "judge": 6.890130353817524,
       "standard": 12.427638871782255
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -31.999999999999996,
       "hard": -45.63106796116505,
       "judge": -47.36842105263158,
       "standard": -46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -0.7665248226950139,
       "hard": -6.760098039215645,
       "judge": 6.645688622754465,
       "standard": -2.297512690355341
      }
     }
    }
   },
   "jevbench_score": 0.018990012754194416,
   "key": "verdict-small",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (verdictml code and the Manav2op/verdict-small checkpoint over intfloat/multilingual-e5-small)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6227544910179641,
    "decisive_rate": 0.44533333333333336,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 72.92000413192591,
     "cost": 100,
     "intelligence": 2.3417095945172277,
     "speed": 89.48731594094681
    },
    "board_release": "v1.6.43",
    "capability": 37.63085686322157,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 119,
    "capability_rank": 147,
    "composite_ci95": {
     "A": [
      0.00034752623458969486,
      0.12857270817562724
     ],
     "B": [
      0.00021950048318305755,
      0.08623222919712645
     ],
     "C": [
      0.0002413376629095103,
      0.08928660289974111
     ]
    },
    "jevbench_score": 0.01899384266854915,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 187,
    "rank": 231,
    "ranks": {
     "A": 231,
     "B": 231,
     "C": 231,
     "capability": 211
    },
    "scores": {
     "A": 0.01899384266854915,
     "B": 0.012337048142708213,
     "C": 0.013190168519825799
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24820413775742053,
     "p50_s_raw": 0.04910206887871027,
     "p95_s_adjusted": 0.4533769000321627,
     "p95_s_raw": 0.15168845001608133
    },
    "views": {
     "Balanced 33:33:33": 0.014681237741460948,
     "Emphasis on Accuracy 60:20:20": 0.008421478349576572,
     "Emphasis on Cost 20:20:60": 0.02342353351655187,
     "Emphasis on Speed 20:60:20": 0.02330657899907492,
     "Intelligence only": 0.005136403075864165
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 253,
   "ranked": true,
   "ranks": {
    "A": 253,
    "B": 253,
    "C": 253,
    "capability": 233
   },
   "repo": "https://github.com/Manavarya09/verdict",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.018990012754194416,
    "B": 0.01233575538253284,
    "C": 0.013187508857079457
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28037555478513243,
    "p50_s_raw": 0.0651877773925662,
    "p95_s_adjusted": 0.47589931823313236,
    "p95_s_raw": 0.1629496591165662
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "intfloat/multilingual-e5-small (118M multilingual bi-encoder) fine-tuned on a typed-decision mix; each option is scored against the rendered state by cosine similarity, with temperature scaling and a conformal abstain set on top. One encoder pass per option, no tokens generated.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.28225057545556,
      76.20018850659073
     ],
     "I": [
      0.6053648936331004,
      4.532806367679299
     ],
     "capability": [
      34.685562893137785,
      39.80647930109692
     ],
     "composite": {
      "A": [
       0.0003475070464105525,
       0.12852579345393955
      ],
      "B": [
       0.00021949435923644138,
       0.08621537426131828
      ],
      "C": [
       0.0002413243377851058,
       0.08925402323190244
      ]
     },
     "seed": 108
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 72.92000413192591,
     "capability": 37.63085686322157,
     "intelligence": 2.3417095945172277
    },
    "per_type": {
     "choice": {
      "accuracy": 0.23733333333333334,
      "calibration": 72.4185625,
      "cc": 2.4658260943701484,
      "cc_by_type": {
       "choice": 2.4658260943701484
      },
      "cc_tiered": 7.0251287835516845,
      "cc_tiered_open": 12.01502895224339,
      "cc_tiered_sealed": 2.035228614859977,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 2.4658260943701484
     },
     "noul": {
      "accuracy": 0.2773333333333333,
      "calibration": 62.35109333333333,
      "cc": -44.53333333333334,
      "cc_by_type": {
       "noul": -44.53333333333334
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2986666666666667,
      "calibration": 83.99035656244443,
      "cc": -2.2494035087719633,
      "cc_by_type": {
       "score": -2.2494035087719633
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 74.70758393102564,
     "capability": 38.932253022553525,
     "intelligence": 3.156922114081418
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.014678186863820933,
    "Emphasis on Accuracy 60:20:20": 0.00842087594735804,
    "Emphasis on Cost 20:20:60": 0.023418873801355072,
    "Emphasis on Speed 20:60:20": 0.023292744564727425,
    "Intelligence only": 0.005136403075864165
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "CodePawl / @anxuanng (founder@codepawl.com)",
   "axes": {
    "calibration": 46.26085085939199,
    "cost": 89.01467380421724,
    "intelligence": 2.2096907014730043,
    "speed": 79.16975860375432
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.43076843750000016,
      "ece_score": 13.846312499999968,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 13.827850749999968,
      "score_before_time_limit": 13.846312499999968,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.6746666666666667,
      "brier": 0.28588573402666656,
      "ece": 0.15048053333333336,
      "n": 375,
      "score": 69.71748295111111,
      "score_before_time_limit": 69.90389333333333,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.736,
      "interval90_width": 0.5186666666666669,
      "n": 375,
      "nrps": 0.21227189381999992,
      "score": 55.23721887706489,
      "score_before_time_limit": 55.38491197566667,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.3400149333333333
     }
    },
    "score": 46.26085085939199
   },
   "capability": 24.235270780432494,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.0010142883035394131,
     0.11149985989593862
    ],
    "B": [
     0.0006462160600988027,
     0.07590635392530619
    ],
    "C": [
     0.0007043668774579258,
     0.07743045826106852
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no public tariff). THE SAME CHOICE THE PUBLISHED quyet-1-0-tiny ROW CARRIES FOR THE IDENTICAL mmBERT-small BACKBONE: DeepInfra base-size encoders at USD 0.005 per 1M input, 0 output - deepinfra:thenlper/gte-base, deepinfra:BAAI/bge-base-en-v1.5, deepinfra:intfloat/e5-base-v2 and deepinfra:sentence-transformers/all-mpnet-base-v2 are all (0.005, 0.0) in the frozen 25 Sep snapshot - x this row's own measured usage.input_tokens (the package's packed length, capped at 4,096). Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, verified in the package source (tacet/decoding.py usage_of sets output_tokens 0; the readout is a softmax over option [MASK] markers from one encoder pass and nothing is generated), the 02 Oct deputy rule for classifier read-outs. RELEASE-LANE LINE, recorded and not acted on, AND IT IS THE QUYET ROWS' CARRIED LINE, NOT A NEW ONE: jhu-clsp/mmBERT-small still has no explicit market-listing decision (pricing_v15.reference() raises), so an UNLISTED_BASE_MODELS entry is the release lane's.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0026020833333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0023236865267433986
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 3,
    "n_over_limit_answered_ok": 3,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Tacet Sonata (CodePawl, 144M packed encoder on mmBERT-small, option-marker softmax in one pass, in-process via the author's tacet package)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. Weights codepawl/tacet-sonata @ 26f447c35881a80020fe557c95ceb8c61bc3fd12, Apache-2.0, public and ungated, loaded from a local snapshot (no network during the run). Runtime tacet==0.3.0 from PyPI, the author's own install line (`pip install \"tacet>=0.3\"`) in an isolated venv (torch 2.14.1+cpu, transformers 5.19.0). ADAPTER: the author's own PR fstandhartinger/jevbench#197 file tacet_local.py at head 76eb75da, byte-identical (sha256 c63a4fe0...), reviewed before running: it maps answers only (noul P(true) -> yes/no; choice and score probabilities used as returned, score keys '0'..'n-1') and reaches nothing but the Hub. It ran through the UNCHANGED v1.6 driver run_v16.py; the only addition, in a wrapper, registers the file as jevbench.adapters.tacet_local and lists it in IN_PROCESS_MODULES as laya_local is listed - lane verification and the runtime egress guard unchanged. Synthetic smoke gate passed before any sealed item was sent (PREREG-R59.md, receipts/SMOKE-tacet-sonata.json). THE 23 LONG ITEMS ARE TRUNCATED, NOT REFUSED, by the package's own code at its default and documented max_length 4,096 (tacet/packing.py keeps the questions and cuts the state; it refuses only states over 200,000 characters, which no item here reaches): exactly 23 records report input_tokens = 4,096 and they are answered and scored. RUN ON SANDY'S SHARED CPU (12-core Zen 2, AVX2, fp32 as the package runs off-accelerator, 8 threads), load average 6.6-24.2 (median ~14) logged every 60 s: the Speed axis is a DECLARED LOWER BOUND, and the author reports raw p50 0.042 s on an RTX 3060 against this run's 0.265 s. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per call, no retry, no batching, offline, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 716D1957 (bh-submit 07d384eb-fc4e-4536-ba61-888567efdf34)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "CPU (12-core Zen 2, AVX2; 8 threads)",
   "intelligence": {
    "I_open": 1.6856027951868566,
    "I_sealed": 2.7337786077591524,
    "base": 2.2096907014730043,
    "excess": -3.6215634550105396,
    "gap": -1.0481758125722958,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 5.05680838556057,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": 11.292859389745214,
       "judge": -6.789716545814108,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -65.21739130434783,
       "judge": -70.0,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 21.489090909090912,
       "hard": -14.746481481481478,
       "judge": 14.17932203389829,
       "standard": -58.12777777777776
      }
     },
     "sealed|choice": {
      "cc": 8.201335823277457,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 16.19244697361616,
       "hard": 4.807692307692292,
       "judge": 5.027932960893876,
       "standard": 15.753171572853812
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -96.0,
       "hard": -47.57281553398058,
       "judge": -52.63157894736843,
       "standard": -69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 2.2302836879432686,
       "hard": -11.765196078431384,
       "judge": -12.453053892215603,
       "standard": -2.9353299492385965
      }
     }
    }
   },
   "jevbench_score": 0.015686432078748844,
   "key": "tacet-sonata",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "codepawl/tacet-sonata@26f447c35881a80020fe557c95ceb8c61bc3fd12",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5619834710743802,
    "decisive_rate": 0.3235294117647059,
    "supported": true,
    "valid": 374
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 46.37837260299999,
     "cost": 89.01467380421724,
     "intelligence": 2.2436273530567146,
     "speed": 80.28211114890141
    },
    "board_release": "v1.6.43",
    "capability": 24.310999978028352,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 132,
    "capability_rank": 162,
    "composite_ci95": {
     "A": [
      0.0012369873259327144,
      0.11178055042619336
     ],
     "B": [
      0.0007893069841145717,
      0.07607180414333671
     ],
     "C": [
      0.0008590189763421627,
      0.07762538224041209
     ]
    },
    "jevbench_score": 0.016405006835540154,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 188,
    "rank": 232,
    "ranks": {
     "A": 232,
     "B": 232,
     "C": 232,
     "capability": 234
    },
    "scores": {
     "A": 0.016405006835540154,
     "B": 0.010748475876643845,
     "C": 0.011392365858013995
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.6127247303724289,
     "p50_s_raw": 0.23136236518621445,
     "p95_s_adjusted": 1.5294084161520003,
     "p95_s_raw": 0.6897042080760002
    },
    "views": {
     "Balanced 33:33:33": 0.012868928732057194,
     "Emphasis on Accuracy 60:20:20": 0.007398330998569957,
     "Emphasis on Cost 20:20:60": 0.02046846868350025,
     "Emphasis on Speed 20:60:20": 0.020367269262702412,
     "Intelligence only": 0.004517645714999133
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 254,
   "ranked": true,
   "ranks": {
    "A": 254,
    "B": 254,
    "C": 254,
    "capability": 256
   },
   "repo": "https://huggingface.co/codepawl/tacet-sonata",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.015686432078748844,
    "B": 0.010273103305863243,
    "C": 0.010893355610242255
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.679555581510067,
    "p50_s_raw": 0.2647777907550335,
    "p95_s_adjusted": 1.7815546773374076,
    "p95_s_raw": 0.8157773386687038
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "jhu-clsp/mmBERT-small",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      40.432909045322795,
      49.803976998599616
     ],
     "I": [
      0.8702131419070752,
      4.3653585772798165
     ],
     "capability": [
      20.99119033296382,
      26.57803527930951
     ],
     "composite": {
      "A": [
       0.0010142883035394131,
       0.11149985989593862
      ],
      "B": [
       0.0006462160600988027,
       0.07590635392530619
      ],
      "C": [
       0.0007043668774579258,
       0.07743045826106852
      ]
     },
     "seed": 143
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 46.26085085939199,
     "capability": 24.235270780432494,
     "intelligence": 2.2096907014730043
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2826666666666667,
      "calibration": 13.827850749999968,
      "cc": 8.263311955893602,
      "cc_by_type": {
       "choice": 8.263311955893602
      },
      "cc_tiered": 6.629072104419014,
      "cc_tiered_open": 5.05680838556057,
      "cc_tiered_sealed": 8.201335823277457,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 8.263311955893602
     },
     "noul": {
      "accuracy": 0.18133333333333335,
      "calibration": 69.71748295111111,
      "cc": -63.733333333333334,
      "cc_by_type": {
       "noul": -63.733333333333334
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.304,
      "calibration": 55.23721887706489,
      "cc": -7.684210526315716,
      "cc_by_type": {
       "score": -7.684210526315716
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 45.61976295895299,
     "capability": 24.54098913610771,
     "intelligence": 3.4622153132624316
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0013333333333332975,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.012298631196612477,
    "Emphasis on Accuracy 60:20:20": 0.007068632431284845,
    "Emphasis on Cost 20:20:60": 0.019574566887244516,
    "Emphasis on Speed 20:60:20": 0.019465551712092424,
    "Intelligence only": 0.004315731879770439
   }
  },
  {
   "adapter_id": "images/laya/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Convai Innovations",
   "axes": {
    "calibration": 63.22326564159259,
    "cost": 84.89402784895724,
    "intelligence": 1.7910341507793501,
    "speed": 72.744382846694
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21969249999999993,
      "ece_score": 56.06150000000001,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 56.06150000000001,
      "score_before_time_limit": 56.06150000000001,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.808,
      "brier": 0.28256342088000014,
      "ece": 0.18899546666666664,
      "n": 375,
      "score": 62.20090666666667,
      "score_before_time_limit": 62.20090666666667,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8826666666666667,
      "interval90_width": 0.7128888888888881,
      "n": 375,
      "nrps": 0.18963512817111114,
      "score": 71.4073902581111,
      "score_before_time_limit": 71.4073902581111,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.19110853333333327
     }
    },
    "score": 63.22326564159259
   },
   "capability": 32.50714989618597,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0002021673844202101,
     0.06710398502970086
    ],
    "B": [
     0.00012765993788192327,
     0.04494779653575676
    ],
    "C": [
     0.00014039401695847923,
     0.04659998960395892
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0031881034482758625
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Laya (Convai Innovations, ModernBERT-large 421M)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 1.8026531464810367,
    "I_sealed": 1.7794151550776638,
    "base": 1.7910341507793501,
    "excess": -2.5501496510348707,
    "gap": 0.023237991403372904,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 5.40795943944311,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.37837837837838,
       "hard": -3.8062283737024116,
       "judge": 16.941331575477918,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -91.30434782608697,
       "judge": -80.0,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 14.020606060606056,
       "hard": -11.245833333333334,
       "judge": -16.895084745762734,
       "standard": -50.04185185185186
      }
     },
     "sealed|choice": {
      "cc": 5.338245465232991,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 1.5593821594856503,
       "hard": -3.846153846153868,
       "judge": 9.683426443202995,
       "standard": 19.078704273925375
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -56.00000000000001,
       "hard": -84.46601941747574,
       "judge": -76.31578947368422,
       "standard": -74.64788732394366
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 4.452695035460996,
       "hard": -11.828823529411746,
       "judge": -8.140898203592851,
       "standard": -0.2669035532995023
      }
     }
    }
   },
   "jevbench_score": 0.008558710204581235,
   "key": "laya",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6111111111111112,
    "decisive_rate": 0.192,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 63.22326564159259,
     "cost": 84.89402784895724,
     "intelligence": 1.7910341507793501,
     "speed": 73.26302890816496
    },
    "board_release": "v1.6.43",
    "capability": 32.50714989618597,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.00020217712264809318,
      0.06712460770140848
     ],
     "B": [
      0.00012766304420524027,
      0.044955197581294834
     ],
     "C": [
      0.00014040077961673136,
      0.046614310903755875
     ]
    },
    "jevbench_score": 0.008560099345041717,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 189,
    "rank": 233,
    "ranks": {
     "A": 233,
     "B": 233,
     "C": 233,
     "capability": 223
    },
    "scores": {
     "A": 0.008560099345041717,
     "B": 0.005540634651880784,
     "C": 0.005944513434056748
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.800616904720664,
     "p50_s_raw": 0.825308452360332,
     "p95_s_adjusted": 2.619846137147397,
     "p95_s_raw": 1.2349230685736985
    },
    "views": {
     "Balanced 33:33:33": 0.006594024627233607,
     "Emphasis on Accuracy 60:20:20": 0.0037729124634726325,
     "Emphasis on Cost 20:20:60": 0.010563725142022983,
     "Emphasis on Speed 20:60:20": 0.010499068205592536,
     "Intelligence only": 0.0022981141246738437
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 255,
   "ranked": true,
   "ranks": {
    "A": 255,
    "B": 255,
    "C": 255,
    "capability": 245
   },
   "repo": "https://huggingface.co/convaiinnovations/laya",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.008558710204581235,
    "B": 0.0055401690319621935,
    "C": 0.005943548753181413
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.0147334575653075,
    "p50_s_raw": 0.9323667287826538,
    "p95_s_adjusted": 2.638421132136136,
    "p95_s_raw": 1.244210566068068
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "ModernBERT-large encoder + option-marker decision head, 421M, RLCD-trained",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      57.988437430874065,
      67.35373438390093
     ],
     "I": [
      0.5052598551452469,
      3.643432797894196
     ],
     "capability": [
      29.732944854873807,
      34.91391985018961
     ],
     "composite": {
      "A": [
       0.0002021673844202101,
       0.06710398502970086
      ],
      "B": [
       0.00012765993788192327,
       0.04494779653575676
      ],
      "C": [
       0.00014039401695847923,
       0.04659998960395892
      ]
     },
     "seed": 56
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 63.22326564159259,
     "capability": 32.50714989618597,
     "intelligence": 1.7910341507793501
    },
    "per_type": {
     "choice": {
      "accuracy": 0.25466666666666665,
      "calibration": 56.06150000000001,
      "cc": 4.682511864952642,
      "cc_by_type": {
       "choice": 4.682511864952642
      },
      "cc_tiered": 5.373102452338051,
      "cc_tiered_open": 5.40795943944311,
      "cc_tiered_sealed": 5.338245465232991,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 4.682511864952642
     },
     "noul": {
      "accuracy": 0.11733333333333333,
      "calibration": 62.20090666666667,
      "cc": -76.53333333333333,
      "cc_by_type": {
       "noul": -76.53333333333333
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2826666666666667,
      "calibration": 71.4073902581111,
      "cc": -7.6405701754385325,
      "cc_by_type": {
       "score": -7.6405701754385325
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 62.99195454673789,
     "capability": 33.098085372560654,
     "intelligence": 3.20421619838342
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.006592925554313149,
    "Emphasis on Accuracy 60:20:20": 0.0037726965510524266,
    "Emphasis on Cost 20:20:60": 0.010562032700695017,
    "Emphasis on Speed 20:60:20": 0.010494054436303035,
    "Intelligence only": 0.0022981141246738437
   }
  },
  {
   "adapter_id": "images/mxbai-rerank-base-v2/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mixedbread",
   "axes": {
    "calibration": 86.60772459153094,
    "cost": 60.33589314238483,
    "intelligence": 1.140707327109041,
    "speed": 92.05600895369167
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06855480504522303,
      "ece_score": 86.28903899095539,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 86.28903899095539,
      "score_before_time_limit": 86.28903899095539,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 1.0,
      "brier": 0.25057053855990563,
      "ece": 0.07454411727378144,
      "n": 375,
      "score": 85.09117654524371,
      "score_before_time_limit": 85.09117654524371,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 1.0,
      "interval90_width": 1.0,
      "n": 375,
      "nrps": 0.17161399632774915,
      "score": 88.44295823839374,
      "score_before_time_limit": 88.44295823839374,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.029763419452187988
     }
    },
    "score": 86.60772459153094
   },
   "capability": 43.87421595931999,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     4.213790740997237e-06,
     0.03217925832474849
    ],
    "B": [
     2.6407563817968135e-06,
     0.021162373242485297
    ],
    "C": [
     2.9262435701369706e-06,
     0.02234670716996423
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-0.6B); no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020996016009852216
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Mixedbread mxbai-rerank-base-v2",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 1.5821442040322362,
    "I_sealed": 0.6992704501858459,
    "base": 1.140707327109041,
    "excess": -1.6905138885918536,
    "gap": 0.8828737538463903,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 4.746432612096709,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -5.081081081081076,
       "hard": -0.03145643284050508,
       "judge": 1.1206328279499007,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -11.030065754992213,
       "hard": -1.5426620524559675,
       "judge": 8.497895555670832,
       "standard": -21.144844232023672
      }
     },
     "sealed|choice": {
      "cc": 0.0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -7.75256817677922,
       "hard": -4.355203619909522,
       "judge": -1.4897579143388995,
       "standard": 0.23401896785319723
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "sealed|score": {
      "cc": 2.0978113505575378,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -4.830772427457819,
       "hard": 1.5240605072453395,
       "judge": 6.561942168694756,
       "standard": 0.013408698983785605
      }
     }
    }
   },
   "jevbench_score": 0.0022737744722840765,
   "key": "mxbai-rerank-base-v2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0.0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.60772459153094,
     "cost": 60.33589314238483,
     "intelligence": 1.140707327109041,
     "speed": 92.58758170313658
    },
    "board_release": "v1.6.43",
    "capability": 43.87421595931999,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 104,
    "capability_rank": 132,
    "composite_ci95": {
     "A": [
      4.213826902762912e-06,
      0.0321843481372824
     ],
     "B": [
      2.6407677436324784e-06,
      0.02116413476940045
     ],
     "C": [
      2.9262686824742456e-06,
      0.022350241762001666
     ]
    },
    "jevbench_score": 0.002273929358844764,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 190,
    "rank": 234,
    "ranks": {
     "A": 234,
     "B": 234,
     "C": 234,
     "capability": 193
    },
    "scores": {
     "A": 0.002273929358844764,
     "B": 0.0014520695133694007,
     "C": 0.001579117610308864
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21285173026844859,
     "p50_s_raw": 0.031425865134224296,
     "p95_s_adjusted": 0.2589194315019995,
     "p95_s_raw": 0.05445971575099974
    },
    "views": {
     "Balanced 33:33:33": 0.0017272292257436567,
     "Emphasis on Accuracy 60:20:20": 0.0009793418989392745,
     "Emphasis on Cost 20:20:60": 0.002776895162759242,
     "Emphasis on Speed 20:60:20": 0.002811535213497941,
     "Intelligence only": 0.0005937213753409678
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 256,
   "ranked": true,
   "ranks": {
    "A": 256,
    "B": 256,
    "C": 256,
    "capability": 214
   },
   "repo": "https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.0022737744722840765,
    "B": 0.0014520189845594442,
    "C": 0.0015790100501972756
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21420394033193588,
    "p50_s_raw": 0.03210197016596794,
    "p95_s_adjusted": 0.2907847651280462,
    "p95_s_raw": 0.0703923825640231
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "494M cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.5709999496809,
      89.28164941290872
     ],
     "I": [
      0.13834674179555617,
      2.8157994354473197
     ],
     "capability": [
      41.27690031534251,
      45.59940972334693
     ],
     "composite": {
      "A": [
       4.213790740997237e-06,
       0.03217925832474849
      ],
      "B": [
       2.6407563817968135e-06,
       0.021162373242485297
      ],
      "C": [
       2.9262435701369706e-06,
       0.02234670716996423
      ]
     },
     "seed": 69
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.60772459153094,
     "capability": 43.87421595931999,
     "intelligence": 1.140707327109041
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2,
      "calibration": 86.28903899095539,
      "cc": -2.3085740268844592,
      "cc_by_type": {
       "choice": -2.3085740268844592
      },
      "cc_tiered": 2.3732163060483544,
      "cc_tiered_open": 4.746432612096709,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.0
     },
     "noul": {
      "accuracy": 0.0,
      "calibration": 85.09117654524371,
      "cc": -100.0,
      "cc_by_type": {
       "noul": -100.0
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.3413333333333333,
      "calibration": 88.44295823839374,
      "cc": 0.3851131600845714,
      "cc_by_type": {
       "score": 0.3851131600845714
      },
      "cc_tiered": 1.0489056752787689,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 2.0978113505575378,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.3851131600845714
     }
    },
    "raw_on_A": {
     "calibration": 84.06228474550946,
     "capability": 42.68728856938789,
     "intelligence": 1.312292393266321
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.0017271100746280466,
    "Emphasis on Accuracy 60:20:20": 0.0009793189142918838,
    "Emphasis on Cost 20:20:60": 0.0027767103766015502,
    "Emphasis on Speed 20:60:20": 0.002810967015237966,
    "Intelligence only": 0.0005937213753409678
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Convai Innovations",
   "axes": {
    "calibration": 33.655846666999985,
    "cost": 82.1551651086044,
    "intelligence": 0.2879184668737643,
    "speed": 76.95139513446136
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.44879250000000004,
      "ece_score": 10.241499999999991,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 10.241499999999991,
      "score_before_time_limit": 10.241499999999991,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.28,
      "brier": 0.4157348964266666,
      "ece": 0.3921216000000001,
      "n": 375,
      "score": 21.575679999999974,
      "score_before_time_limit": 21.575679999999974,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.896,
      "interval90_width": 0.7111111111111107,
      "n": 375,
      "nrps": 0.18850373331333325,
      "score": 69.150360001,
      "score_before_time_limit": 69.150360001,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.21424453333333335
     }
    },
    "score": 33.655846666999985
   },
   "capability": 16.971882566936873,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     0.0,
     0.016095436642730325
    ],
    "B": [
     0.0,
     0.01067226036496501
    ],
    "C": [
     0.0,
     0.011177386557451619
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003933940886699508
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Laya multilingual",
   "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 0.3687761432977424,
    "I_sealed": 0.20706079044978623,
    "base": 0.2879184668737643,
    "excess": -2.4116722895902876,
    "gap": 0.16171535284795616,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 1.1063284298932272,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.702702702702708,
       "hard": -13.243158225857174,
       "judge": 9.03098220171391,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -30.434782608695656,
       "judge": -50.0,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 13.373939393939372,
       "hard": 2.3077777777777753,
       "judge": -8.492881355932202,
       "standard": -37.68148148148147
      }
     },
     "sealed|choice": {
      "cc": 0.6211823713493587,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 2.889660778952063,
       "hard": -4.86425339366518,
       "judge": 5.959031657355701,
       "standard": 2.451040768567572
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 0.0,
       "hard": -30.097087378640776,
       "judge": -36.8421052631579,
       "standard": -29.5774647887324
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -0.387943262411361,
       "hard": -0.8122549019607739,
       "judge": -1.6932335329341575,
       "standard": -3.1987817258883444
      }
     }
    }
   },
   "jevbench_score": 3.759412198193081e-05,
   "key": "laya-multilingual",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.4962962962962963,
    "decisive_rate": 0.72,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 33.655846666999985,
     "cost": 82.1551651086044,
     "intelligence": 0.2879184668737643,
     "speed": 77.82044954853706
    },
    "board_release": "v1.6.43",
    "capability": 16.971882566936873,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 136,
    "capability_rank": 166,
    "composite_ci95": {
     "A": [
      0,
      0.016100086645554724
     ],
     "B": [
      0,
      0.010673895641907892
     ],
     "C": [
      0,
      0.01118061572607967
     ]
    },
    "jevbench_score": 3.7595668430455566e-05,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 191,
    "rank": 235,
    "ranks": {
     "A": 235,
     "B": 235,
     "C": 235,
     "capability": 238
    },
    "scores": {
     "A": 3.7595668430455566e-05,
     "B": 2.368099374667976e-05,
     "C": 2.6108103076705254e-05
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.8654558090260253,
     "p50_s_raw": 0.3577279045130126,
     "p95_s_adjusted": 1.908579030103283,
     "p95_s_raw": 0.8792895150516415
    },
    "views": {
     "Balanced 33:33:33": 2.843624303166903e-05,
     "Emphasis on Accuracy 60:20:20": 1.5873606797432713e-05,
     "Emphasis on Cost 20:20:60": 4.7066204506481234e-05,
     "Emphasis on Speed 20:60:20": 4.704809343432325e-05,
     "Intelligence only": 9.547035876864281e-06
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 257,
   "ranked": true,
   "ranks": {
    "A": 257,
    "B": 257,
    "C": 257,
    "capability": 260
   },
   "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/multilingual",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.759412198193081e-05,
    "B": 2.368050288566296e-05,
    "C": 2.610702915411862e-05
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.953681958001107,
    "p50_s_raw": 0.4018409790005535,
    "p95_s_adjusted": 2.1157138055132236,
    "p95_s_raw": 0.9828569027566117
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, multilingual/; mmBERT-base 322M, max_len 1024. Distinct from the published English root checkpoint. Fresh run from item 1 on 29 Sep; interrupted earlier runs were not stitched. In-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. The release owner independently adopted mmBERT as unlisted in the frozen pricing snapshots; the labelled same-class encoder estimate has zero cost delta against BAAI/bge-m3.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      28.738769095461095,
      38.12861857012684
     ],
     "I": [
      0.0,
      2.2482651295080083
     ],
     "capability": [
      14.566920276697426,
      19.746046606060705
     ],
     "composite": {
      "A": [
       0.0,
       0.016095436642730325
      ],
      "B": [
       0.0,
       0.01067226036496501
      ],
      "C": [
       0.0,
       0.011177386557451619
      ]
     },
     "seed": 57
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 33.655846666999985,
     "capability": 16.971882566936873,
     "intelligence": 0.2879184668737643
    },
    "per_type": {
     "choice": {
      "accuracy": 0.21733333333333332,
      "calibration": 10.241499999999991,
      "cc": -0.09188825630196516,
      "cc_by_type": {
       "choice": -0.09188825630196516
      },
      "cc_tiered": 0.863755400621293,
      "cc_tiered_open": 1.1063284298932272,
      "cc_tiered_sealed": 0.6211823713493587,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.0
     },
     "noul": {
      "accuracy": 0.35733333333333334,
      "calibration": 21.575679999999974,
      "cc": -28.53333333333333,
      "cc_by_type": {
       "noul": -28.53333333333333
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.296,
      "calibration": 69.150360001,
      "cc": -1.8657280701754209,
      "cc_by_type": {
       "score": -1.8657280701754209
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 32.7172206036182,
     "capability": 17.460072535779506,
     "intelligence": 2.202924467940814
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.8435063409660868e-05,
    "Emphasis on Accuracy 60:20:20": 1.587338624446757e-05,
    "Emphasis on Cost 20:20:60": 4.706426554955254e-05,
    "Emphasis on Speed 20:60:20": 4.7042281517922134e-05,
    "Intelligence only": 9.547035876864281e-06
   }
  },
  {
   "adapter_id": "images/certo/spec.json:certo_v15.CertoLocalV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AltSlate Labs",
   "axes": {
    "calibration": 86.14595667337036,
    "cost": 96.9130801359906,
    "intelligence": 0.20278658812856143,
    "speed": 92.44797254495796
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06436937500000006,
      "ece_score": 87.12612499999999,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 87.12612499999999,
      "score_before_time_limit": 87.12612499999999,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.9893333333333333,
      "brier": 0.25290737730666696,
      "ece": 0.08273039999999988,
      "n": 375,
      "score": 83.45392000000002,
      "score_before_time_limit": 83.45392000000002,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9946666666666667,
      "interval90_width": 0.9942222222222223,
      "n": 375,
      "nrps": 0.17416936626444454,
      "score": 87.8578250201111,
      "score_before_time_limit": 87.8578250201111,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0343370666666668
     }
    },
    "score": 86.14595667337036
   },
   "capability": 43.17437163074946,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0,
     0.00442750939229265
    ],
    "B": [
     0.0,
     0.0028324122193567084
    ],
    "C": [
     0.0,
     0.0030746593002032303
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0012673522167487683
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Certo v1 (AltSlate Labs)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 0.12963682017919345,
    "I_sealed": 0.27593635607792943,
    "base": 0.20278658812856143,
    "excess": -2.71968717833698,
    "gap": -0.14629953589873598,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 1.855929537590448,
       "judge": -18.655240606460122,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -91.30434782608697,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 0.3889104605375804,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 5.111515151515156,
       "hard": -0.6524999999999892,
       "judge": 1.8106779661016925,
       "standard": -2.0222222222222364
      }
     },
     "sealed|choice": {
      "cc": 0.6168084160817238,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -6.42228955731281,
       "hard": -4.86425339366518,
       "judge": -0.5586592178770757,
       "standard": 16.861682473210998
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "sealed|score": {
      "cc": 0.21100065215206465,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -0.7955319148936102,
       "hard": 1.9993137254901705,
       "judge": -1.0858682634730998,
       "standard": -0.9170558375635629
      }
     }
    }
   },
   "jevbench_score": 1.3254504525307007e-05,
   "key": "certo",
   "last_measured_on": "2026-10-02",
   "licence": "MIT",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.25,
    "decisive_rate": 0.010666666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.14595667337036,
     "cost": 96.9130801359906,
     "intelligence": 0.20278658812856143,
     "speed": 92.53493469610669
    },
    "board_release": "v1.6.43",
    "capability": 43.17437163074946,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 106,
    "capability_rank": 134,
    "composite_ci95": {
     "A": [
      0,
      0.004427570625789243
     ],
     "B": [
      0,
      0.0028324322673094096
     ],
     "C": [
      0,
      0.003074701823464752
     ]
    },
    "jevbench_score": 1.3254531668229452e-05,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 192,
    "rank": 236,
    "ranks": {
     "A": 236,
     "B": 236,
     "C": 236,
     "capability": 195
    },
    "scores": {
     "A": 1.3254531668229452e-05,
     "B": 8.311485824043969e-06,
     "C": 9.204535880714898e-06
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23407380385324358,
     "p50_s_raw": 0.042036901926621795,
     "p95_s_adjusted": 0.2383162835612893,
     "p95_s_raw": 0.04415814178064466
    },
    "views": {
     "Balanced 33:33:33": 9.964199692814982e-06,
     "Emphasis on Accuracy 60:20:20": 5.551453514139297e-06,
     "Emphasis on Cost 20:20:60": 1.653808420915537e-05,
     "Emphasis on Speed 20:60:20": 1.6534837758955257e-05,
     "Intelligence only": 3.3356285030110653e-06
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 258,
   "ranked": true,
   "ranks": {
    "A": 258,
    "B": 258,
    "C": 258,
    "capability": 217
   },
   "repo": "https://huggingface.co/altslate/certo-decision-model",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 1.3254504525307007e-05,
    "B": 8.311477285661616e-06,
    "C": 9.204517031463201e-06
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23454320020973682,
    "p50_s_raw": 0.04227160010486841,
    "p95_s_adjusted": 0.24264977835118767,
    "p95_s_raw": 0.046324889175593846
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "ModernBERT-large with a per-option query/scoring head, ~400M parameters",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.84167298672317,
      89.02661690205646
     ],
     "I": [
      0.0,
      1.4261709005081855
     ],
     "capability": [
      40.54746106861931,
      44.91215541214051
     ],
     "composite": {
      "A": [
       0.0,
       0.00442750939229265
      ],
      "B": [
       0.0,
       0.0028324122193567084
      ],
      "C": [
       0.0,
       0.0030746593002032303
      ]
     },
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.14595667337036,
     "capability": 43.17437163074946,
     "intelligence": 0.20278658812856143
    },
    "per_type": {
     "choice": {
      "accuracy": 0.21733333333333332,
      "calibration": 87.12612499999999,
      "cc": -0.09188825630196516,
      "cc_by_type": {
       "choice": -0.09188825630196516
      },
      "cc_tiered": 0.3084042080408619,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.6168084160817238,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.0
     },
     "noul": {
      "accuracy": 0.0026666666666666666,
      "calibration": 83.45392000000002,
      "cc": -99.46666666666667,
      "cc_by_type": {
       "noul": -99.46666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.27466666666666667,
      "calibration": 87.8578250201111,
      "cc": 0.4315701754385892,
      "cc_by_type": {
       "score": 0.4315701754385892
      },
      "cc_tiered": 0.2999555563448225,
      "cc_tiered_open": 0.3889104605375804,
      "cc_tiered_sealed": 0.21100065215206465,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.4315701754385892
     }
    },
    "raw_on_A": {
     "calibration": 84.87407902121794,
     "capability": 42.46944871565377,
     "intelligence": 0.06481841008959673
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.964179240079037e-06,
    "Emphasis on Accuracy 60:20:20": 5.551449704956405e-06,
    "Emphasis on Cost 20:20:60": 1.6538050403588868e-05,
    "Emphasis on Speed 20:60:20": 1.6534736382482662e-05,
    "Intelligence only": 3.3356285030110653e-06
   }
  },
  {
   "adapter_id": "images/clm-8b/spec.json:clm_adapter.CLMAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Contrastive-LM (Kwok, Kang, Suresh, Saad-Falcon, Pavone, Ré, Mirhoseini)",
   "axes": {
    "calibration": 40.03295405037662,
    "cost": 50.501662240594015,
    "intelligence": 0.13075619159747265,
    "speed": 91.7219631344249
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.3993510766836583,
      "ece_score": 20.129784663268335,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 20.10294495038398,
      "score_before_time_limit": 20.129784663268335,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.4906666666666667,
      "brier": 0.26050031793956074,
      "ece": 0.18460455903608142,
      "n": 375,
      "score": 63.079088192783715,
      "score_before_time_limit": 63.079088192783715,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.688,
      "interval90_width": 0.5184444444444443,
      "n": 375,
      "nrps": 0.2846437031480146,
      "score": 36.91682900796216,
      "score_before_time_limit": 36.91682900796216,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.48850985834637106
     }
    },
    "score": 40.03295405037662
   },
   "capability": 20.081855120987044,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     0.0,
     0.010706273666911133
    ],
    "B": [
     0.0,
     0.007039957182729574
    ],
    "C": [
     0.0,
     0.007434912268688287
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.044662660714285714
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "CLM-8B (Contrastive-LM, clm-latest)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 0.2615123831949453,
    "I_sealed": 0.0,
    "base": 0.13075619159747265,
    "excess": -2.3118752592432985,
    "gap": 0.2615123831949453,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 0.7845371495848359,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -16.75675675675675,
       "hard": -7.5810003145643154,
       "judge": 16.941331575477918,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -56.52173913043479,
       "judge": -19.999999999999996,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -21.94096032005719,
       "hard": -1.9559175706170606,
       "judge": -11.452540725941729,
       "standard": -95.63691251131768
      }
     },
     "sealed|choice": {
      "cc": 0.0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -6.42228955731281,
       "hard": -4.355203619909522,
       "judge": 0.3724394785847514,
       "standard": 7.993595270353507
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -31.999999999999996,
       "hard": -33.98058252427184,
       "judge": -31.57894736842105,
       "standard": -9.859154929577462
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -10.75981978481908,
       "hard": -29.341375883900135,
       "judge": -28.232663703069093,
       "standard": -55.86396111215961
      }
     }
    }
   },
   "jevbench_score": 3.5510448593503677e-06,
   "key": "clm-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code and CLM-v0.1-8B head); Qwen3-8B encoder Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6910994764397905,
    "decisive_rate": 0.5093333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 40.041900621338065,
     "cost": 50.501662240594015,
     "intelligence": 0.13075619159747265,
     "speed": 92.63548383949004
    },
    "board_release": "v1.6.43",
    "capability": 20.086328406467768,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 133,
    "capability_rank": 163,
    "composite_ci95": {
     "A": [
      0,
      0.010708585645302497
     ],
     "B": [
      0,
      0.007040756824095157
     ],
     "C": [
      0,
      0.007436517809237845
     ]
    },
    "jevbench_score": 3.5510969933114796e-06,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 193,
    "rank": 237,
    "ranks": {
     "A": 237,
     "B": 237,
     "C": 237,
     "capability": 235
    },
    "scores": {
     "A": 3.5510969933114796e-06,
     "B": 2.2274698008760027e-06,
     "C": 2.466039578688528e-06
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19238147847354412,
     "p50_s_raw": 0.02119073923677206,
     "p95_s_adjusted": 0.2833272576332092,
     "p95_s_raw": 0.0666636288166046
    },
    "views": {
     "Balanced 33:33:33": 2.67198512793619e-06,
     "Emphasis on Accuracy 60:20:20": 1.488390058441499e-06,
     "Emphasis on Cost 20:20:60": 4.430457758214625e-06,
     "Emphasis on Speed 20:60:20": 4.4408219452037955e-06,
     "Intelligence only": 8.942249433748766e-07
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 259,
   "ranked": true,
   "ranks": {
    "A": 259,
    "B": 259,
    "C": 259,
    "capability": 257
   },
   "repo": "https://github.com/Contrastive-LM/CLM",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.5510448593503677e-06,
    "B": 2.227453390729157e-06,
    "C": 2.4660033745488667e-06
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22330221589654684,
    "p50_s_raw": 0.03665110794827342,
    "p95_s_adjusted": 0.30123862307518723,
    "p95_s_raw": 0.0756193115375936
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      35.37121914730655,
      44.441323364577684
     ],
     "I": [
      0.0,
      1.9510432807841849
     ],
     "capability": [
      17.69959643896708,
      22.526253333840447
     ],
     "composite": {
      "A": [
       0.0,
       0.010706273666911133
      ],
      "B": [
       0.0,
       0.007039957182729574
      ],
      "C": [
       0.0,
       0.007434912268688287
      ]
     },
     "seed": 22
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 40.03295405037662,
     "capability": 20.081855120987044,
     "intelligence": 0.13075619159747265
    },
    "per_type": {
     "choice": {
      "accuracy": 0.20533333333333334,
      "calibration": 20.10294495038398,
      "cc": -1.6265168667052297,
      "cc_by_type": {
       "choice": -1.6265168667052297
      },
      "cc_tiered": 0.39226857479241795,
      "cc_tiered_open": 0.7845371495848359,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.0
     },
     "noul": {
      "accuracy": 0.352,
      "calibration": 63.079088192783715,
      "cc": -29.600000000000005,
      "cc_by_type": {
       "noul": -29.600000000000005
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.26666666666666666,
      "calibration": 36.91682900796216,
      "cc": -29.29970040054666,
      "cc_by_type": {
       "score": -29.29970040054666
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 38.47490938800186,
     "capability": 19.538705486965597,
     "intelligence": 0.6025015859293292
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0013333333333332975,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.6719477148185174e-06,
    "Emphasis on Accuracy 60:20:20": 1.488383093064189e-06,
    "Emphasis on Cost 20:20:60": 4.4303960412854775e-06,
    "Emphasis on Speed 20:60:20": 4.440635932355796e-06,
    "Intelligence only": 8.942249433748766e-07
   }
  },
  {
   "adapter_id": "images/bge-reranker-v2-m3/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "BAAI",
   "axes": {
    "calibration": 88.10046932372114,
    "cost": 59.325605115070715,
    "intelligence": 0.06512222845080358,
    "speed": 93.5678736531519
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.027473000546109336,
      "ece_score": 94.50539989077814,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 94.50539989077814,
      "score_before_time_limit": 94.50539989077814,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 1.0,
      "brier": 0.25096294008809744,
      "ece": 0.0684901503152521,
      "n": 375,
      "score": 86.30196993694959,
      "score_before_time_limit": 86.30196993694959,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 1.0,
      "interval90_width": 1.0,
      "n": 375,
      "nrps": 0.1745560185107233,
      "score": 83.49403814343572,
      "score_before_time_limit": 83.49403814343572,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07778160931028115
     }
    },
    "score": 88.10046932372114
   },
   "capability": 44.08279577608597,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     0.0,
     0.008573175268890313
    ],
    "B": [
     0.0,
     0.005538532607600911
    ],
    "C": [
     0.0,
     0.0059535939367293825
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: base-model market reference (deepinfra:BAAI/bge-m3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.022688885467980296
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "BAAI bge-reranker-v2-m3",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 0.018692443590466585,
    "I_sealed": 0.11155201331114058,
    "base": 0.06512222845080358,
    "excess": -2.666247212158918,
    "gap": -0.092859569720674,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 0.05607733077139976,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 7.518087448883308,
       "judge": -14.700065919578117,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 5.306919494288776,
       "hard": -1.440659315203363,
       "judge": -2.980317227364382,
       "standard": -2.397579483854617
      }
     },
     "sealed|choice": {
      "cc": 0.33465603993342175,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -5.092010937846398,
       "hard": -0.791855203619931,
       "judge": 2.234636871508399,
       "standard": 2.451040768567572
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 2.319622122935927,
       "hard": -0.38388489349363564,
       "judge": 0.07005074041354087,
       "standard": -0.8557298803223867
      }
     }
    }
   },
   "jevbench_score": 4.4076704742616216e-07,
   "key": "bge-reranker-v2-m3",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0.0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.10046932372114,
     "cost": 59.325605115070715,
     "intelligence": 0.06512222845080358,
     "speed": 93.86976236118387
    },
    "board_release": "v1.6.43",
    "capability": 44.08279577608597,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 102,
    "capability_rank": 130,
    "composite_ci95": {
     "A": [
      0,
      0.008573668302290527
     ],
     "B": [
      0,
      0.005538697219146524
     ],
     "C": [
      0,
      0.005953936321035087
     ]
    },
    "jevbench_score": 4.4076803151681546e-07,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 194,
    "rank": 238,
    "ranks": {
     "A": 238,
     "B": 238,
     "C": 238,
     "capability": 191
    },
    "scores": {
     "A": 4.4076803151681546e-07,
     "B": 2.758281497029271e-07,
     "C": 3.060889107755663e-07
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1841465687379241,
     "p50_s_raw": 0.01707328436896205,
     "p95_s_adjusted": 0.2227717584464699,
     "p95_s_raw": 0.03638587922323495
    },
    "views": {
     "Balanced 33:33:33": 3.308199423158738e-07,
     "Emphasis on Accuracy 60:20:20": 1.8400822603920523e-07,
     "Emphasis on Cost 20:20:60": 5.501608977147286e-07,
     "Emphasis on Speed 20:60:20": 5.506039727293195e-07,
     "Intelligence only": 1.1047086427994778e-07
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 260,
   "ranked": true,
   "ranks": {
    "A": 260,
    "B": 260,
    "C": 260,
    "capability": 212
   },
   "repo": "https://huggingface.co/BAAI/bge-reranker-v2-m3",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 4.4076704742616216e-07,
    "B": 2.758278413965028e-07,
    "C": 3.060882273792793e-07
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18669292386621236,
    "p50_s_raw": 0.018346461933106184,
    "p95_s_adjusted": 0.23555090506561097,
    "p95_s_raw": 0.042775452532805486
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "568M multilingual cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.14061191725726,
      90.57274307882632
     ],
     "I": [
      0.0,
      1.7895886627489592
     ],
     "capability": [
      41.80444040346114,
      45.59879463409008
     ],
     "composite": {
      "A": [
       0.0,
       0.008573175268890313
      ],
      "B": [
       0.0,
       0.005538532607600911
      ],
      "C": [
       0.0,
       0.0059535939367293825
      ]
     },
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.10046932372114,
     "capability": 44.08279577608597,
     "intelligence": 0.06512222845080358
    },
    "per_type": {
     "choice": {
      "accuracy": 0.22133333333333333,
      "calibration": 94.50539989077814,
      "cc": 0.4196546138324576,
      "cc_by_type": {
       "choice": 0.4196546138324576
      },
      "cc_tiered": 0.19536668535241075,
      "cc_tiered_open": 0.05607733077139976,
      "cc_tiered_sealed": 0.33465603993342175,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.4196546138324576
     },
     "noul": {
      "accuracy": 0.0,
      "calibration": 86.30196993694959,
      "cc": -100.0,
      "cc_by_type": {
       "noul": -100.0
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2826666666666667,
      "calibration": 83.49403814343572,
      "cc": -0.18199548701309798,
      "cc_by_type": {
       "score": -0.18199548701309798
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 85.43472005366168,
     "capability": 43.01708899298245,
     "intelligence": 0.5994579323032229
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.308192031583021e-07,
    "Emphasis on Accuracy 60:20:20": 1.8400808883101353e-07,
    "Emphasis on Cost 20:20:60": 5.501596711672631e-07,
    "Emphasis on Speed 20:60:20": 5.506002871741509e-07,
    "Intelligence only": 1.1047086427994778e-07
   }
  },
  {
   "adapter_id": "images/gte-reranker-modernbert-base/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba-NLP",
   "axes": {
    "calibration": 83.4556233651092,
    "cost": 69.26965431492843,
    "intelligence": 0.0,
    "speed": 93.50537133277697
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0790764439307023,
      "ece_score": 84.18471121385954,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 84.18471121385954,
      "score_before_time_limit": 84.18471121385954,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 1.0,
      "brier": 0.25231019651494707,
      "ece": 0.08962639509020176,
      "n": 375,
      "score": 82.07472098195964,
      "score_before_time_limit": 82.07472098195964,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 1.0,
      "interval90_width": 1.0,
      "n": 375,
      "nrps": 0.17667560258952839,
      "score": 84.10743789950841,
      "score_before_time_limit": 84.10743789950841,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07058781971015167
     }
    },
    "score": 83.4556233651092
   },
   "capability": 41.7278116825546,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     0.0,
     0.0036625510569188633
    ],
    "B": [
     0.0,
     0.002344053723577124
    ],
    "C": [
     0.0,
     0.0025434382339714324
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:thenlper/gte-base); no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010576570197044334
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Alibaba GTE Reranker ModernBERT-base",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 0.0,
    "I_sealed": 0.0,
    "base": 0.0,
    "excess": -2.573387642438244,
    "gap": 0.0,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": -3.8062283737024116,
       "judge": -6.789716545814108,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -1.920002214440264,
       "hard": -1.3526386292960213,
       "judge": 1.2639659476201492,
       "standard": 0.3612908498087819
      }
     },
     "sealed|choice": {
      "cc": 0.0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -5.092010937846398,
       "hard": -6.391402714932147,
       "judge": -5.214152700186197,
       "standard": -7.525557334647108
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -1.317182445372178,
       "hard": 0.03511410774166279,
       "judge": -0.2802076730282099,
       "standard": 0.30690766431314964
      }
     }
    }
   },
   "jevbench_score": 0.0,
   "key": "gte-reranker-modernbert-base",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0.0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.4556233651092,
     "cost": 69.26965431492843,
     "intelligence": 0,
     "speed": 93.76120791185616
    },
    "board_release": "v1.6.43",
    "capability": 41.7278116825546,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 110,
    "capability_rank": 138,
    "composite_ci95": {
     "A": [
      0,
      0.003662687483805933
     ],
     "B": [
      0,
      0.0023440984279851505
     ],
     "C": [
      0,
      0.002543532974865231
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 195,
    "rank": 239,
    "ranks": {
     "A": 239,
     "B": 239,
     "C": 239,
     "capability": 202
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19835810186341404,
     "p50_s_raw": 0.024179050931707025,
     "p95_s_adjusted": 0.2120455999393016,
     "p95_s_raw": 0.031022799969650804
    },
    "views": {
     "Balanced 33:33:33": 0,
     "Emphasis on Accuracy 60:20:20": 0,
     "Emphasis on Cost 20:20:60": 0,
     "Emphasis on Speed 20:60:20": 0,
     "Intelligence only": 0
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 261,
   "ranked": true,
   "ranks": {
    "A": 261,
    "B": 261,
    "C": 261,
    "capability": 225
   },
   "repo": "https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.0,
    "B": 0.0,
    "C": 0.0
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19940753942355513,
    "p50_s_raw": 0.024703769711777568,
    "p95_s_adjusted": 0.2237284903414547,
    "p95_s_raw": 0.03686424517072736
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "149M ModernBERT cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.59536793549519,
      87.51314141306226
     ],
     "I": [
      0.0,
      1.339161261347401
     ],
     "capability": [
      39.372795544944715,
      43.89015023558699
     ],
     "composite": {
      "A": [
       0.0,
       0.0036625510569188633
      ],
      "B": [
       0.0,
       0.002344053723577124
      ],
      "C": [
       0.0,
       0.0025434382339714324
      ]
     },
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.4556233651092,
     "capability": 41.7278116825546,
     "intelligence": 0.0
    },
    "per_type": {
     "choice": {
      "accuracy": 0.17866666666666667,
      "calibration": 84.18471121385954,
      "cc": -5.036802667601379,
      "cc_by_type": {
       "choice": -5.036802667601379
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.0
     },
     "noul": {
      "accuracy": 0.0,
      "calibration": 82.07472098195964,
      "cc": -100.0,
      "cc_by_type": {
       "noul": -100.0
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.304,
      "calibration": 84.10743789950841,
      "cc": -0.24811073820258933,
      "cc_by_type": {
       "score": -0.24811073820258933
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 88.93979403901187,
     "capability": 44.469897019505936,
     "intelligence": 0.0
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.0,
    "Emphasis on Accuracy 60:20:20": 0.0,
    "Emphasis on Cost 20:20:60": 0.0,
    "Emphasis on Speed 20:60:20": 0.0,
    "Intelligence only": 0.0
   }
  },
  {
   "adapter_id": "images/mirror/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bluusun",
   "axes": {
    "calibration": 51.61277239297377,
    "cost": 89.27300805786774,
    "intelligence": 0.0,
    "speed": 74.03935684279031
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2204942558257705,
      "ece_score": 55.90114883484589,
      "mean_tvd": null,
      "n": 379,
      "n_ece_hard": 100,
      "n_tvd": 0,
      "score": 55.90114883484589,
      "score_before_time_limit": 55.90114883484589,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.07773851590106007,
      "brier": 0.3825718867287919,
      "ece": 0.37640864622953524,
      "n": 283,
      "score": 24.718270754092952,
      "score_before_time_limit": 24.718270754092952,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9471544715447154,
      "interval90_width": 0.8113143631436313,
      "n": 246,
      "nrps": 0.18235540291655192,
      "score": 74.21889758998248,
      "score_before_time_limit": 74.21889758998248,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1666333226418992
     }
    },
    "score": 51.61277239297377
   },
   "capability": 25.806386196486883,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0,
     9.853496598484777e-09
    ],
    "B": [
     0.0,
     6.1614500141756846e-09
    ],
    "C": [
     0.0,
     6.8427059711699815e-09
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0022780665024630543
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Mirror",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 0.0,
    "I_sealed": 0.0,
    "base": 0.0,
    "excess": -2.573387642438244,
    "gap": 0.0,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": -7.5810003145643154,
       "judge": -14.700065919578117,
       "standard": -5.4834658853076625
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0.0,
       "hard": -73.91304347826086,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 14.02621656810803,
       "hard": -86.48465684738163,
       "judge": -131.29938184642268,
       "standard": -61.876452149889175
      }
     },
     "sealed|choice": {
      "cc": 0.0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 14.86216835414975,
       "hard": -16.063348416289617,
       "judge": -12.662942271880798,
       "standard": -6.417046434289924
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 12.00000000000001,
       "hard": -33.98058252427184,
       "judge": -15.789473684210531,
       "standard": -7.042253521126762
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -1.6525187057042423,
       "hard": -113.52957658615944,
       "judge": -73.84737475308705,
       "standard": -29.673128809498152
      }
     }
    }
   },
   "jevbench_score": 0.0,
   "key": "mirror",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 wrapper and Mirror release; upstream DeBERTa/model assets retain their own terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5977011494252874,
    "decisive_rate": 0.9222614840989399,
    "supported": true,
    "valid": 283
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 51.61277239297377,
     "cost": 89.27300805786774,
     "intelligence": 0,
     "speed": 73.47211485993975
    },
    "board_release": "v1.6.43",
    "capability": 25.806386196486883,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0,
      9.853474730694675e-09
     ],
     "B": [
      0,
      6.161443172813211e-09
     ],
     "C": [
      0,
      6.8426907852046335e-09
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 196,
    "rank": 240,
    "ranks": {
     "A": 240,
     "B": 240,
     "C": 240,
     "capability": 233
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.4516588259488343,
     "p50_s_raw": 0.6508294129744172,
     "p95_s_adjusted": 3.096876988979056,
     "p95_s_raw": 1.473438494489528
    },
    "views": {
     "Balanced 33:33:33": 0,
     "Emphasis on Accuracy 60:20:20": 0,
     "Emphasis on Cost 20:20:60": 0,
     "Emphasis on Speed 20:60:20": 0,
     "Intelligence only": 0
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 262,
   "ranked": true,
   "ranks": {
    "A": 262,
    "B": 262,
    "C": 262,
    "capability": 255
   },
   "repo": "https://github.com/Bluusun/Decision-API",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.0,
    "B": 0.0,
    "C": 0.0
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.2716480298899113,
    "p50_s_raw": 0.5608240149449557,
    "p95_s_adjusted": 3.10239715129137,
    "p95_s_raw": 1.476198575645685
   },
   "status": {
    "answered_ok": 908,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Mirror DeBERTa-v3-large 436M classification-span scorer",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      42.56771416804497,
      56.74054715693854
     ],
     "I": [
      0.0,
      0.01760273906633102
     ],
     "capability": [
      21.283857084022486,
      28.426215016960548
     ],
     "composite": {
      "A": [
       0.0,
       9.853496598484777e-09
      ],
      "B": [
       0.0,
       6.1614500141756846e-09
      ],
      "C": [
       0.0,
       6.8427059711699815e-09
      ]
     },
     "seed": 68
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 51.61277239297377,
     "capability": 25.806386196486883,
     "intelligence": 0.0
    },
    "per_type": {
     "choice": {
      "accuracy": 0.156,
      "calibration": 55.90114883484589,
      "cc": -7.935545598363106,
      "cc_by_type": {
       "choice": -7.935545598363106
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.0
     },
     "noul": {
      "accuracy": 0.416,
      "calibration": 24.718270754092952,
      "cc": -16.800000000000004,
      "cc_by_type": {
       "noul": -16.800000000000004
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.18666666666666668,
      "calibration": 74.21889758998248,
      "cc": -72.82982094179775,
      "cc_by_type": {
       "score": -72.82982094179775
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 48.60190522372855,
     "capability": 24.300952611864275,
     "intelligence": 0.0
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.4946666666666667,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.2453333333333333,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.344,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.0,
    "Emphasis on Accuracy 60:20:20": 0.0,
    "Emphasis on Cost 20:20:60": 0.0,
    "Emphasis on Speed 20:60:20": 0.0,
    "Intelligence only": 0.0
   }
  },
  {
   "adapter_id": "images/open-jev-json-canvas-joshuasp/spec.json:joshuasp_adapter.JoshuaSPCanvasAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "JoshuaSP",
   "axes": {
    "calibration": 0.0,
    "cost": 49.295219536461964,
    "intelligence": 61.19221807027989,
    "speed": 85.23407766636957
   },
   "calibration": {
    "parts": {},
    "score": 0.0
   },
   "capability": 30.596109035139946,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0,
     0.0
    ],
    "B": [
     0.0,
     0.0
    ],
    "C": [
     0.0,
     0.0
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04899585591133005
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open Jev JSON Canvas (JoshuaSP)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 61.36340599482574,
    "I_sealed": 61.02103014573405,
    "base": 61.19221807027989,
    "excess": -2.2310117933465556,
    "gap": 0.34237584909168817,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 60.21617418117772,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 49.040578798364265,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 72.15458937198068,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 51.71945443131884,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.54545454545454,
       "hard": 38.88888888888889,
       "judge": 67.79661016949152,
       "standard": 51.85185185185186
      }
     },
     "sealed|choice": {
      "cc": 68.74482965663773,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 60.80316742081447,
       "judge": 71.13594040968344,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 64.47674149136002,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 45.63106796116505,
       "judge": 76.3157894736842,
       "standard": 74.64788732394365
      }
     },
     "sealed|score": {
      "cc": 49.84151928920441,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.92198581560282,
       "hard": 46.81372549019608,
       "judge": 50.29940119760479,
       "standard": 44.6700507614213
      }
     }
    }
   },
   "jevbench_score": 0.0,
   "key": "open-jev-json-canvas-joshuasp",
   "last_measured_on": "2026-10-02",
   "licence": "MIT code; Apache-2.0 DiffusionGemma weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8490566037735849,
    "decisive_rate": 1.0,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 0,
     "cost": 49.295219536461964,
     "intelligence": 61.19221807027989,
     "speed": 85.53332924416175
    },
    "board_release": "v1.6.43",
    "capability": 30.596109035139946,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 129,
    "capability_rank": 158,
    "composite_ci95": {
     "A": [
      0,
      0
     ],
     "B": [
      0,
      0
     ],
     "C": [
      0,
      0
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 197,
    "rank": 241,
    "ranks": {
     "A": 241,
     "B": 241,
     "C": 241,
     "capability": 227
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.44667022498324516,
     "p50_s_raw": 0.14833511249162257,
     "p95_s_adjusted": 0.6261524326633662,
     "p95_s_raw": 0.23807621633168308
    },
    "views": {
     "Balanced 33:33:33": 60.34909628466887,
     "Emphasis on Accuracy 60:20:20": 59.99813997005745,
     "Emphasis on Cost 20:20:60": 54.674045042588084,
     "Emphasis on Speed 20:60:20": 67.78111653907041,
     "Intelligence only": 59.47929292694184
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 263,
   "ranked": true,
   "ranks": {
    "A": 263,
    "B": 263,
    "C": 263,
    "capability": 249
   },
   "repo": "https://github.com/JoshuaSP/open-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.0,
    "B": 0.0,
    "C": 0.0
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.46701108301058414,
    "p50_s_raw": 0.15850554150529206,
    "p95_s_adjusted": 0.6416010205633939,
    "p95_s_raw": 0.24580051028169692
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "label",
    "noul": "label",
    "score": "label"
   },
   "underlying": "google/diffusiongemma-26B-A4B-it BF16, one-step JSON canvas",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      0.0,
      0.0
     ],
     "I": [
      54.979058028212734,
      65.63316060204029
     ],
     "capability": [
      27.489529014106367,
      32.816580301020146
     ],
     "composite": {
      "A": [
       0.0,
       0.0
      ],
      "B": [
       0.0,
       0.0
      ],
      "C": [
       0.0,
       0.0
      ]
     },
     "seed": 76
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 0.0,
     "capability": 30.596109035139946,
     "intelligence": 61.19221807027989
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7493333333333333,
      "calibration": null,
      "cc": 67.9433134715762,
      "cc_by_type": {
       "choice": 67.9433134715762
      },
      "cc_tiered": 64.48050191890772,
      "cc_tiered_open": 60.21617418117772,
      "cc_tiered_sealed": 68.74482965663773,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.9433134715762
     },
     "noul": {
      "accuracy": 0.84,
      "calibration": null,
      "cc": 68.0,
      "cc_by_type": {
       "noul": 68.0
      },
      "cc_tiered": 68.31566543167034,
      "cc_tiered_open": 72.15458937198068,
      "cc_tiered_sealed": 64.47674149136002,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 68.0
     },
     "score": {
      "accuracy": 0.624,
      "calibration": null,
      "cc": 50.6140350877193,
      "cc_by_type": {
       "score": 50.6140350877193
      },
      "cc_tiered": 50.780486860261625,
      "cc_tiered_open": 51.71945443131884,
      "cc_tiered_sealed": 49.84151928920441,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.6140350877193
     }
    },
    "raw_on_A": {
     "calibration": 0.0,
     "capability": 31.028102732437542,
     "intelligence": 62.056205464875084
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 60.297872730137286,
    "Emphasis on Accuracy 60:20:20": 59.967751855484444,
    "Emphasis on Cost 20:20:60": 54.6488096472127,
    "Emphasis on Speed 20:60:20": 67.66490703085857,
    "Intelligence only": 59.47929292694184
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "APUS AI-LAB (@cryptoSUN2049 / findoo2049@gmail.com)",
   "axes": {
    "calibration": 67.91594971090585,
    "cost": null,
    "intelligence": 52.808647095759525,
    "speed": 80.338990145866
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21087790603358122,
      "ece_score": 57.82441879328375,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 57.82441879328375,
      "score_before_time_limit": 57.82441879328375,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0835579514824798,
      "brier": 0.14666984115552173,
      "ece": 0.13630441067160756,
      "n": 371,
      "score": 72.73911786567848,
      "score_before_time_limit": 72.73911786567848,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8238482384823849,
      "interval90_width": 0.22018970189701897,
      "n": 369,
      "nrps": 0.10709067085641949,
      "score": 73.18431247375533,
      "score_before_time_limit": 73.18431247375533,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.21461153983423703
     }
    },
    "score": 67.91594971090585
   },
   "capability": 60.36229840333269,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED, and deliberately not invented. pricing_v15.reference() RAISES on the exact declared base Qwen/Qwen3.5-35B-A3B ('base model lacks an explicit market-listing decision'). Run 46's bongard-mini rule applies unchanged: the frozen module raises rather than estimating, and the FROZEN-TABLE ROW is the release lane's (board #11 / #10132) - not this job's and not Florian's, because his standing 24 Sep 20:30 rule already decides the principle and the 26 Sep rule fixes the method. Complete but unpriced - shown, not ranked. RELEASE-LANE LINE, and a tight one: the dated snapshot ALREADY lists the exact base as qwen/qwen3.5-35b-a3b at (0.3125, 1.25), and BASE_REFERENCES ALREADY carries 'Qwen/Qwen3.5-35B-A3B-Base' -> 'qwen/qwen3.5-35b-a3b'. Only the non--Base (instruct) spelling of the same model is missing, so every Qwen3.5-35B-A3B derivative raises today. BECAUSE OUTPUT TOKENS ARE 0 BY CONSTRUCTION this row's Cost depends on the input rate alone, so the number is one multiplication over tokens the raw keeps; r56/receipts/COST-ARITHMETIC-R56.json carries it already computed.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "APUS-OpenJev-v1-35B-A3B (merged bf16 checkpoint-5949 MoE on Qwen3.5-35B-A3B, their own native runtime, full depth, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights apus-ailab/APUS-OpenJev-v1-35B-A3B@090388abe2237117cc3b4df6f4250e81cbb36864, Apache-2.0, 66.99 GiB bf16 merged weights, checkpoint-5949, Qwen3_5MoeForConditionalGeneration (40 layers, 256 experts, 8 active), declared base Qwen/Qwen3.5-35B-A3B, full depth 40 layers (their depth_config declares exit_depth 20 / full_depth 40). accelerate>=1.10 additionally, per this repo's own requirements.txt. STACK: the authors' own requirements.txt, unmodified - torch==2.8.0 and transformers==5.16.1. The transformers pin is THEIRS, not a workaround: OpenJet.__init__ raises 'Layer execution is audited for transformers==5.16.1' on any other version. Served behind a TRANSPORT SHIM ONLY (r56/serve/apus_systemone_r56.py): loaded with THEIR OpenJet.from_pretrained(device=cuda:0, dtype=bfloat16) and every request handed to THEIR decide(), returning THEIR format_response() output. No contributor file edited. effort='high', the full-depth path, their decide() default and the configuration their own headline Frozen80 number is quoted on; effort='low' is not measured and no routing between the two is invented. One H100 PCIe 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. MAPPING (run 56's, not the authors'; pre-registered in PREREG-R56 s2.2 before any item was sent and repeated here because it is part of the number): their runtime accepts primitive in {choice, noul, score_level} with 2-16 candidates labelled A-P. choice -> their 'choice' primitive, candidates = the item's own labels as id and its own criteria[label] as description, in labels order; their format_response returns probabilities keyed by candidate id, i.e. by the JevBench label already, so nothing is rewritten. All 750 choice items carry 3-9 options, inside their 2-16 window. noul -> their 'noul' primitive; their contract REQUIRES the canonical yes/no criteria there, so the item's own true/false description strings are NOT delivered - the proposition itself travels verbatim in instructions, which is what their noul is built to judge. score -> their 'choice' primitive over the ordinal levels, because their own RUNTIME.md rules score_level out as an ordinal Score API ('score_level is the judgement of a single proposition, it cannot be treated as a complete ordinal Score API'); the 4-5 ordered levels become candidates with id = the level index and description = the item's own level text, which carries the order. Recording all 375 score items as misses instead would understate a system whose published interface answers them. CALIBRATION IS MEASURED AS THE AUTHORS SHIP IT AND THEY DISCLOSED IT IN ADVANCE: their response carries calibrated: False and both their card and RUNTIME.md state the probabilities are an uncalibrated relative softmax over the offered candidates. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, NOT BY ESTIMATE: decide() runs one forward pass with logits_to_keep=1 and reads candidate logits; nothing is generated anywhere in that path. Input tokens are their own prompt_tokens field. THE 23 LONG ITEMS: their compile() raises ValueError('input exceeds runtime limit; no truncation permitted') above 8,192 tokens and their RUNTIME.md states it in advance. Recorded invalid. This is the first row in the series whose refusal on those items is guaranteed in the author's own code with an explicit no-truncation promise. add-requests run 56, GitHub #105, benchmarkheaven.com/submit ref AE709BBB",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100 PCIe",
   "intelligence": {
    "I_open": 54.71860410185919,
    "I_sealed": 50.898690089659866,
    "base": 52.808647095759525,
    "excess": 1.2465263697610829,
    "gap": 3.8199140121993267,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 60.14590221055973,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965714,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 64.24775707384403,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 50.0,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 39.762153021173845,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.86605733086419,
       "hard": 29.028440368940068,
       "judge": 66.44632488030888,
       "standard": 15.151368382093667
      }
     },
     "sealed|choice": {
      "cc": 59.02771188966037,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 51.131221719457,
       "judge": 59.031657355679705,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 49.333250807861994,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 32.038834951456316,
       "judge": 57.89473684210527,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 44.33510757145725,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.35456392472854,
       "hard": 33.80629692129137,
       "judge": 52.81314502924963,
       "standard": 39.66594450846479
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "apus-openjev-v1-35b-a3b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "apus-ailab/APUS-OpenJev-v1-35B-A3B@090388abe2237117cc3b4df6f4250e81cbb36864",
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.8529411764705882,
    "decisive_rate": 0.9164420485175202,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 67.91594971090585,
     "cost": null,
     "intelligence": 52.808647095759525,
     "speed": 80.50610262660216
    },
    "board_release": "v1.6.43",
    "capability": 60.36229840333269,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "capability": 66
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.859432493080385,
     "p50_s_raw": 0.3547162465401925,
     "p95_s_adjusted": 1.0355664409697056,
     "p95_s_raw": 0.4427832204848528
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": 52.808647095759525
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "capability": 66
   },
   "repo": "https://huggingface.co/apus-ailab/APUS-OpenJev-v1-35B-A3B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8697583910310641,
    "p50_s_raw": 0.35987919551553205,
    "p95_s_adjusted": 1.0634139621281065,
    "p95_s_raw": 0.4567069810640532
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-35B-A3B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      63.096906433455544,
      71.75588820395149
     ],
     "I": [
      45.94798828612946,
      57.24766533646037
     ],
     "capability": [
      55.374556366190006,
      63.9661711269007
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 67.91594971090585,
     "capability": 60.36229840333269,
     "intelligence": 52.808647095759525
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 57.82441879328375,
      "cc": 59.92914183947025,
      "cc_by_type": {
       "choice": 59.92914183947025
      },
      "cc_tiered": 59.58680705011005,
      "cc_tiered_open": 60.14590221055973,
      "cc_tiered_sealed": 59.02771188966037,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.92914183947025
     },
     "noul": {
      "accuracy": 0.7733333333333333,
      "calibration": 72.73911786567848,
      "cc": 54.666666666666664,
      "cc_by_type": {
       "noul": 54.666666666666664
      },
      "cc_tiered": 56.790503940853014,
      "cc_tiered_open": 64.24775707384403,
      "cc_tiered_sealed": 49.333250807861994,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 54.666666666666664
     },
     "score": {
      "accuracy": 0.5946666666666667,
      "calibration": 73.18431247375533,
      "cc": 43.44124606258307,
      "cc_by_type": {
       "score": 43.44124606258307
      },
      "cc_tiered": 42.04863029631555,
      "cc_tiered_open": 39.762153021173845,
      "cc_tiered_sealed": 44.33510757145725,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.44124606258307
     }
    },
    "raw_on_A": {
     "calibration": 68.85867558536114,
     "capability": 63.06963064948997,
     "intelligence": 57.280585713618805
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": 52.808647095759525
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Babak Barazandeh (bbarazandeh@cribl.io, GitHub @bbarazandeh-crypto)",
   "axes": {
    "calibration": 82.13857452060611,
    "cost": null,
    "intelligence": 38.67355961154223,
    "speed": 80.91147986788948
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1281668719369918,
      "ece_score": 74.36662561260164,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 74.36662561260164,
      "score_before_time_limit": 74.36662561260164,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.28800000000000003,
      "brier": 0.1440167151699402,
      "ece": 0.07805321715454966,
      "n": 375,
      "score": 84.38935656909007,
      "score_before_time_limit": 84.38935656909007,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9146666666666666,
      "interval90_width": 0.45133333333333325,
      "n": 375,
      "nrps": 0.10596156489230459,
      "score": 87.65974138012665,
      "score_before_time_limit": 87.65974138012665,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07042180375258127
     }
    },
    "score": 82.13857452060611
   },
   "capability": 60.40606706607417,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED - NOT A MEASUREMENT FAILURE. The base is disclosed (Qwen/Qwen3.5-4B-Base; reference() resolves it to deepinfra:Qwen/Qwen3.5-4B at 0.03/0.15), but the endpoint reports usage.input_tokens ONLY - no output_tokens on any answer - and the system is API-only, so 'nothing is generated' cannot be verified in its code; the 02 Oct deputy rule keeps an unverified null unpriced. The frozen scorer therefore gives no Cost axis and no composite; the row is reported on Capability. RELEASE-LANE LINE: if the author confirms (or the release lane accepts) a one-pass readout with 0 output tokens, the row prices by one multiplication from the raw at 0.03/M input (receipts/COST-ALTERNATIVES-R59.json).",
    "common_basis": null,
    "kind": "estimated",
    "usage_estimated_rows": 1500,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "BB-Qwen3.5-4B-LoRA (Babak Barazandeh, LoRA on Qwen3.5-4B-Base, hosted /v1/systemone)",
   "endpoint_condition": "Hosted full-set API measured 2026-10-07; submitted LoRA checkpoint-030500 on Qwen3.5-4B-Base, weights not released. Serial requests over author ngrok tunnel, no retry; latency includes transit. Complete 1500/1500 answers. Input usage is reported but output usage is absent, so Cost and composite remain unpriced; Capability is reported. Exposure ledger recorded before dispatch.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 40.54310423897533,
    "I_sealed": 36.80401498410914,
    "base": 38.67355961154223,
    "excess": 1.16570161242795,
    "gap": 3.739089254866194,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 54.16068441724518,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 52.815350739226176,
       "judge": 64.40342781806196,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 28.017253278122844,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 39.45137502155798,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.5237907386309,
       "hard": 38.20598063967837,
       "judge": 35.150958941159246,
       "standard": 50.35658004737882
      }
     },
     "sealed|choice": {
      "cc": 53.65959260673962,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 49.09502262443438,
       "judge": 50.65176908752329,
       "standard": 60.09360758714129
      }
     },
     "sealed|noul": {
      "cc": 19.15862307210663,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": -4.854368932038833,
       "judge": 26.315789473684205,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 37.59382927348118,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 61.46292444619053,
       "hard": 28.220009445462267,
       "judge": 41.5085830641898,
       "standard": 38.53479065710138
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "bb-qwen3.5-4b-lora",
   "last_measured_on": "2026-10-07",
   "licence": null,
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "checkpoint-030500 (author declaration); base revision 1001bb4d",
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.8764044943820225,
    "decisive_rate": 0.712,
    "supported": true,
    "valid": 375
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.13857452060611,
     "cost": null,
     "intelligence": 38.67355961154223,
     "speed": 81.57813729165589
    },
    "board_release": "v1.6.43",
    "capability": 60.40606706607417,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "capability": 65
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.6954362019896507,
     "p50_s_raw": 0.6954362019896507,
     "p95_s_adjusted": 0.9998364746570587,
     "p95_s_raw": 0.9998364746570587
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": 23.13675425207104
    }
   },
   "provenance": {
    "carried_metadata_from": "Current registry and GitHub #156; no credential or private endpoint URL copied",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "capability": 65
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/156",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7166377417743206,
    "p50_s_raw": 0.7166377417743206,
    "p95_s_adjusted": 1.131233705580234,
    "p95_s_raw": 1.131233705580234
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B-Base",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.70685263219829,
      84.35075129046632
     ],
     "I": [
      32.59884752255904,
      43.02764829916808
     ],
     "capability": [
      55.59198238994904,
      62.87426662252782
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.13857452060611,
     "capability": 60.40606706607417,
     "intelligence": 38.67355961154223
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6493333333333333,
      "calibration": 74.36662561260164,
      "cc": 55.15474171821564,
      "cc_by_type": {
       "choice": 55.15474171821564
      },
      "cc_tiered": 53.9101385119924,
      "cc_tiered_open": 54.16068441724518,
      "cc_tiered_sealed": 53.65959260673962,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.15474171821564
     },
     "noul": {
      "accuracy": 0.624,
      "calibration": 84.38935656909007,
      "cc": 24.8,
      "cc_by_type": {
       "noul": 24.8
      },
      "cc_tiered": 23.587938175114736,
      "cc_tiered_open": 28.017253278122844,
      "cc_tiered_sealed": 19.15862307210663,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.8
     },
     "score": {
      "accuracy": 0.5413333333333333,
      "calibration": 87.65974138012665,
      "cc": 38.101218047454275,
      "cc_by_type": {
       "score": 38.101218047454275
      },
      "cc_tiered": 38.52260214751958,
      "cc_tiered_open": 39.45137502155798,
      "cc_tiered_sealed": 37.59382927348118,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 38.101218047454275
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": 23.13675425207104
   }
  },
  {
   "adapter_id": "run/wity-1-always.spec.json:wity_reasoning_adapter.WityReasoningAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 91.0421772337923,
     "cost": 41.78292309262718,
     "intelligence": 79.49960112075699,
     "speed": 74.36141604169042
    },
    "label": "Base-model reference price (base undisclosed at the author's request)",
    "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens",
    "usd_per_1000": 0.08721059113300492
   },
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Wity (alphanimble)",
   "axes": {
    "calibration": 87.14296044619107,
    "cost": 58.8349720563948,
    "intelligence": 67.11413626626955,
    "speed": 68.5105981536613
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07469390266334129,
      "ece_score": 85.06121946733174,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 82.906335240826,
      "score_before_time_limit": 85.06121946733174,
      "time_limit_failed_share": 0.025333333333333333
     },
     "noul": {
      "abstention_rate": 0.07999999999999996,
      "brier": 0.07909001541945995,
      "ece": 0.05007116234318407,
      "n": 375,
      "score": 88.78595729761167,
      "score_before_time_limit": 89.98576753136318,
      "time_limit_failed_share": 0.013333333333333334
     },
     "score": {
      "interval90_coverage": 0.9493333333333334,
      "interval90_width": 0.21622222222222207,
      "n": 375,
      "nrps": 0.051449317612291365,
      "score": 89.73658880013555,
      "score_before_time_limit": 92.70308760344582,
      "time_limit_failed_share": 0.032,
      "top_ece": 0.047244465159396204
     }
    },
    "score": 87.14296044619107
   },
   "capability": 77.12854835623031,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     66.94020306458522,
     70.08957190157982
    ],
    "B": [
     65.87195241242884,
     70.26842127647677
    ],
    "C": [
     66.94020306458522,
     70.08957190157982
    ]
   },
   "cost": {
    "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.074026736
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.023559582938388626
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 36,
    "n_over_limit_answered_ok": 36,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "wity-1 (Wity, reasoning always)",
   "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=always; labelled configuration variant.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.5119499160984,
    "I_sealed": 66.7163226164407,
    "base": 67.11413626626955,
    "excess": -1.7777603427805482,
    "gap": 0.7956272996576956,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 81.71302445902249,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 71.68921044353571,
       "judge": 88.13447593935398,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 58.719806763285035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 50.0,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 62.10301852598769,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.97327196189137,
       "hard": 60.98101749360573,
       "judge": 66.87830350526892,
       "standard": 50.24896640387794
      }
     },
     "sealed|choice": {
      "cc": 73.810452177689,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 85.3669351858695,
       "hard": 64.36651583710406,
       "judge": 77.6536312849162,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 66.72801427882574,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 92.0,
       "hard": 47.57281553398058,
       "judge": 71.05263157894737,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 59.6105013928074,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 83.95007438141664,
       "hard": 56.1573234045365,
       "judge": 56.336971033260674,
       "standard": 59.257366414364654
      }
     }
    }
   },
   "jevbench_score": 69.00315679345155,
   "key": "wity-1-always",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.9212827988338192,
    "decisive_rate": 0.927027027027027,
    "supported": true,
    "valid": 370
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.25002486738025,
     "cost": 58.8349720563948,
     "intelligence": 69.25112551166924,
     "speed": 71.20296182763504
    },
    "board_release": "v1.6.43",
    "capability": 79.25057518952474,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      68.6626707423781,
      71.62549018189077
     ],
     "B": [
      67.67306476984521,
      71.90451506429471
     ],
     "C": [
      68.6626707423781,
      71.62549018189077
     ]
    },
    "jevbench_score": 70.56449594359258,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": null,
    "scores": {
     "A": 70.56449594359258,
     "B": 70.29785155628596,
     "C": 70.56449594359258
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.6885605081915855,
     "p50_s_raw": 1.6885605081915855,
     "p95_s_adjusted": 4.489388496428728,
     "p95_s_raw": 4.489388496428728
    },
    "views": {
     "Balanced 33:33:33": 65.96124511263525,
     "Emphasis on Accuracy 60:20:20": 67.23896089186712,
     "Emphasis on Cost 20:20:60": 62.91315123993267,
     "Emphasis on Speed 20:60:20": 67.96251148296399,
     "Intelligence only": 69.25112551166924
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": null,
   "ranked": false,
   "repo": "https://wity.alphanimble.com/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.00315679345155,
    "B": 68.61689325174478,
    "C": 69.00315679345155
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.9074126482009888,
    "p50_s_raw": 1.9074126482009888,
    "p95_s_adjusted": 7.3874665338545995,
    "p95_s_raw": 7.3874665338545995
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Base model: undisclosed (author request)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.60096012448746,
      88.72350025719587
     ],
     "I": [
      61.704668526310996,
      71.10143835839432
     ],
     "capability": [
      72.6594538006522,
      79.50426160694899
     ],
     "composite": {
      "A": [
       66.94020306458522,
       70.08957190157982
      ],
      "B": [
       65.87195241242884,
       70.26842127647677
      ],
      "C": [
       66.94020306458522,
       70.08957190157982
      ]
     },
     "seed": 112
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.14296044619107,
     "capability": 77.12854835623031,
     "intelligence": 67.11413626626955
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 82.906335240826,
      "cc": 75.61645652359255,
      "cc_by_type": {
       "choice": 75.61645652359255
      },
      "cc_tiered": 77.76173831835575,
      "cc_tiered_open": 81.71302445902249,
      "cc_tiered_sealed": 73.810452177689,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.61645652359255
     },
     "noul": {
      "accuracy": 0.8426666666666667,
      "calibration": 88.78595729761167,
      "cc": 68.53333333333333,
      "cc_by_type": {
       "noul": 68.53333333333333
      },
      "cc_tiered": 62.72391052105539,
      "cc_tiered_open": 58.719806763285035,
      "cc_tiered_sealed": 66.72801427882574,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 68.53333333333333
     },
     "score": {
      "accuracy": 0.7813333333333333,
      "calibration": 89.73658880013555,
      "cc": 61.602397935774036,
      "cc_by_type": {
       "score": 61.602397935774036
      },
      "cc_tiered": 60.85675995939755,
      "cc_tiered_open": 62.10301852598769,
      "cc_tiered_sealed": 59.6105013928074,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 61.602397935774036
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.02533333333333332,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.03200000000000003,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.52588954983555,
    "Emphasis on Accuracy 60:20:20": 65.5368572234174,
    "Emphasis on Cost 20:20:60": 62.12233224407349,
    "Emphasis on Speed 20:60:20": 66.06282613355656,
    "Intelligence only": 67.11413626626955
   }
  },
  {
   "adapter_id": "run/wity-1-off.spec.json:wity_reasoning_adapter.WityReasoningAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 83.45289319036758,
     "cost": 41.78292309262718,
     "intelligence": 60.40492797149856,
     "speed": 87.66397534612332
    },
    "label": "Base-model reference price (base undisclosed at the author's request)",
    "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens",
    "usd_per_1000": 0.08721059113300492
   },
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Wity (alphanimble)",
   "axes": {
    "calibration": 81.82569250484788,
    "cost": 58.8349720563948,
    "intelligence": 44.11116118075945,
    "speed": 87.61214901323027
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10943535174564842,
      "ece_score": 78.11292965087031,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 78.11292965087031,
      "score_before_time_limit": 78.11292965087031,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.20266666666666666,
      "brier": 0.14300007101908263,
      "ece": 0.10507128193494888,
      "n": 375,
      "score": 78.98574361301023,
      "score_before_time_limit": 78.98574361301023,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9493333333333334,
      "interval90_width": 0.5226666666666664,
      "n": 375,
      "nrps": 0.09960810647445889,
      "score": 88.37840425066312,
      "score_before_time_limit": 88.37840425066312,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06641190425613937
     }
    },
    "score": 81.82569250484788
   },
   "capability": 62.968426842803666,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     32.15286775090439,
     65.37736939946349
    ],
    "B": [
     28.787777128825557,
     61.33770222198112
    ],
    "C": [
     22.32838038257249,
     45.400950971849646
    ]
   },
   "cost": {
    "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.074026736
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.023559582938388626
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "wity-1 (Wity, reasoning off)",
   "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=off; labelled configuration variant. The first approximately 70 decisions ran alongside two other streams; the remainder ran alone.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.233518722452544,
    "I_sealed": 46.98880363906636,
    "base": 44.11116118075945,
    "excess": -8.32867255905206,
    "gap": -5.755284916613817,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 60.211735029468755,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 50.92796476879523,
       "judge": 64.40342781806196,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 29.155969634230512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 34.332851503658375,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.55807900089734,
       "hard": 31.747271308753067,
       "judge": 45.35548485808892,
       "standard": 6.857448113203668
      }
     },
     "sealed|choice": {
      "cc": 60.2170501413345,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 49.60407239819003,
       "judge": 60.89385474860336,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 37.60945396446127,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 20.388349514563096,
       "judge": 42.10526315789473,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 43.13990681140334,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.76401506785865,
       "hard": 42.31306223924574,
       "judge": 40.34902565317462,
       "standard": 38.66786356483397
      }
     }
    }
   },
   "jevbench_score": 49.18145184761536,
   "key": "wity-1-off",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.8762541806020067,
    "decisive_rate": 0.7973333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.82569250484788,
     "cost": 58.8349720563948,
     "intelligence": 44.11116118075945,
     "speed": 89.55912090143326
    },
    "board_release": "v1.6.43",
    "capability": 62.968426842803666,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      32.27069947350198,
      65.6460656322266
     ],
     "B": [
      28.862486308497225,
      61.52715969788266
     ],
     "C": [
      22.410207967709706,
      45.58754557793514
     ]
    },
    "jevbench_score": 49.37499462500363,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": null,
    "scores": {
     "A": 49.37499462500363,
     "B": 45.396938654270464,
     "C": 34.28819071180808
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.28173011541366577,
     "p50_s_raw": 0.28173011541366577,
     "p95_s_adjusted": 0.39287521950900556,
     "p95_s_raw": 0.39287521950900556
    },
    "views": {
     "Balanced 33:33:33": 45.93426190954148,
     "Emphasis on Accuracy 60:20:20": 40.46467602098598,
     "Emphasis on Cost 20:20:60": 45.877374451824245,
     "Emphasis on Speed 20:60:20": 53.189857817529536,
     "Intelligence only": 34.3325026440474
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": null,
   "ranked": false,
   "repo": "https://wity.alphanimble.com/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 49.18145184761536,
    "B": 45.265912980714305,
    "C": 34.153786005288445
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3454003408551216,
    "p50_s_raw": 0.3454003408551216,
    "p95_s_adjusted": 0.501721066609025,
    "p95_s_raw": 0.501721066609025
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Base model: undisclosed (author request)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.68787622816288,
      83.1859182390883
     ],
     "I": [
      37.039067859171155,
      49.769681165865016
     ],
     "capability": [
      57.45466681572874,
      65.81540071488247
     ],
     "composite": {
      "A": [
       32.15286775090439,
       65.37736939946349
      ],
      "B": [
       28.787777128825557,
       61.33770222198112
      ],
      "C": [
       22.32838038257249,
       45.400950971849646
      ]
     },
     "seed": 113
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.82569250484788,
     "capability": 62.968426842803666,
     "intelligence": 44.11116118075945
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6933333333333334,
      "calibration": 78.11292965087031,
      "cc": 60.781713289694295,
      "cc_by_type": {
       "choice": 60.781713289694295
      },
      "cc_tiered": 60.214392585401626,
      "cc_tiered_open": 60.211735029468755,
      "cc_tiered_sealed": 60.2170501413345,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.781713289694295
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 78.98574361301023,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 33.38271179934589,
      "cc_tiered_open": 29.155969634230512,
      "cc_tiered_sealed": 37.60945396446127,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.5706666666666667,
      "calibration": 88.37840425066312,
      "cc": 43.067156612799565,
      "cc_by_type": {
       "score": 43.067156612799565
      },
      "cc_tiered": 38.73637915753086,
      "cc_tiered_open": 34.332851503658375,
      "cc_tiered_sealed": 43.13990681140334,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.067156612799565
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 45.71112779287298,
    "Emphasis on Accuracy 60:20:20": 40.360542246694095,
    "Emphasis on Cost 20:20:60": 45.74356490122393,
    "Emphasis on Speed 20:60:20": 52.65413381303676,
    "Intelligence only": 34.3325026440474
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "mrmps (@michael_chomsky)",
   "axes": {
    "calibration": 91.33,
    "cost": 58.89,
    "intelligence": 65.16,
    "speed": 78.21
   },
   "calibration": {
    "score": 91.33
   },
   "capability": 78.25,
   "class": "jev-service",
   "composite_ci95": {
    "A": [
     67.79,
     74.01
    ],
    "B": [
     65.25,
     74.13
    ],
    "C": [
     59.49,
     74.01
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): operator list price: higher of 19 Sep plan cost and 26 Sep usage tariff USD 0.042/M input (rule 1.2); no exact base-model floor applies",
    "kind": "estimate",
    "usd_per_1000": 0.02345172413793104
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "classifier.dev (fast tier)",
   "endpoint_condition": "the operator's hosted API",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 67.78,
    "I_sealed": 57.39,
    "base": 65.16,
    "per_type_split": {
     "open|choice": {
      "cc": 79.2975,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.5766,
       "judge": 76.269,
       "standard": 84.9309
      }
     },
     "open|noul": {
      "cc": 57.942,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.5217,
       "judge": 40.0,
       "standard": 66.6667
      }
     },
     "open|score": {
      "cc": 66.1139,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 72.6364,
       "hard": 64.3333,
       "judge": 75.5254,
       "standard": 52.2963
      }
     },
     "sealed|choice": {
      "cc": 74.0974,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 83.3179,
       "hard": 68.9755,
       "judge": 68.3417,
       "standard": 88.3646
      }
     },
     "sealed|noul": {
      "cc": 47.1925,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 36.3636,
       "judge": 50.0,
       "standard": 50.0
      }
     },
     "sealed|score": {
      "cc": 50.894,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 79.6944,
       "hard": 31.0482,
       "judge": 48.7273,
       "standard": 79.4356
      }
     }
    }
   },
   "jevbench_score": 71.35,
   "key": "classifier-dev-fast",
   "last_measured_on": "2026-10-06",
   "licence": "MIT (code); hosted service",
   "listing": "wrapper",
   "method": "v1.7",
   "not_ranked_because": "Wrapper that serves Jev: listed, never ranked.",
   "open": "no",
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.33,
     "cost": 58.89,
     "intelligence": 65.16,
     "speed": 79.54
    },
    "board_release": "v1.6.43",
    "capability": 78.25,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      68.03,
      74.31
     ],
     "B": [
      65.43,
      74.36
     ],
     "C": [
      59.7,
      74.31
     ]
    },
    "jevbench_score": 71.62,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 71.62,
     "B": 70.23,
     "C": 71.62
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.806,
     "p50_s_raw": 0.806,
     "p95_s_adjusted": 1.381,
     "p95_s_raw": 1.381
    },
    "views": null
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://classifier.dev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.35,
    "B": 70.02,
    "C": 71.35
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.98,
    "p50_s_raw": 0.98,
    "p95_s_adjusted": 1.542,
    "p95_s_raw": 1.542
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "cc0da2fbc545cd44056fce4dbc8b1e17b6adbbe73146aa905c9abcfa6b0d3a87"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "rayrain-srp (ZooWork)",
   "axes": {
    "calibration": 93.17,
    "cost": 64.41,
    "intelligence": 47.77,
    "speed": 87.14
   },
   "calibration": {
    "score": 93.17
   },
   "capability": 70.47,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     39.53,
     72.11
    ],
    "B": [
     35.42,
     68.6
    ],
    "C": [
     27.45,
     66.05
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.03/M input, 0/M output (spec instinct.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.015355329949238578
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Instinct (ZooWork, Qwen3.8-27B)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 50.26,
    "I_sealed": 40.14,
    "base": 47.77,
    "per_type_split": {
     "open|choice": {
      "cc": 76.6889,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.2162,
       "hard": 69.8018,
       "judge": 80.2241,
       "standard": 77.3964
      }
     },
     "open|noul": {
      "cc": 22.4893,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 21.7391,
       "judge": 10.0,
       "standard": 11.1111
      }
     },
     "open|score": {
      "cc": 51.5916,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.5514,
       "hard": 43.6306,
       "judge": 66.2661,
       "standard": 40.522
      }
     },
     "sealed|choice": {
      "cc": 69.9997,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 88.8786,
       "hard": 61.2194,
       "judge": 68.3417,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 14.607,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 64.7059,
       "hard": 9.0909,
       "judge": -10.0,
       "standard": 37.5
      }
     },
     "sealed|score": {
      "cc": 35.8134,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 62.6995,
       "hard": 22.3344,
       "judge": 24.153,
       "standard": 66.8189
      }
     }
    }
   },
   "jevbench_score": 62.23,
   "key": "instinct",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary service over Apache-2.0 base weights (Qwen3.8-27B)",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 23,
    "axes": {
     "calibration": 93.17,
     "cost": 64.41,
     "intelligence": 47.77,
     "speed": 87.2
    },
    "board_release": "v1.6.43",
    "capability": 70.47,
    "capability_api_board_rank": 19,
    "capability_open_board_rank": null,
    "capability_rank": 45,
    "composite_ci95": {
     "A": [
      39.54,
      72.12
     ],
     "B": [
      35.42,
      68.61
     ],
     "C": [
      27.46,
      66.05
     ]
    },
    "jevbench_score": 62.24,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 55,
    "ranks": {
     "A": 55,
     "B": 58,
     "C": 61,
     "capability": 76
    },
    "scores": {
     "A": 62.24,
     "B": 57.34,
     "C": 43.22
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.292,
     "p50_s_raw": 0.292,
     "p95_s_adjusted": 0.652,
     "p95_s_raw": 0.652
    },
    "views": null
   },
   "rank": 61,
   "ranked": true,
   "ranks": {
    "A": 61,
    "B": 65,
    "C": 71,
    "capability": 88
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/69",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 62.23,
    "B": 57.33,
    "C": 43.22
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.296,
    "p50_s_raw": 0.296,
    "p95_s_adjusted": 0.653,
    "p95_s_raw": 0.653
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "83e1676aa839fe8666ffd6623d701c421d5841083daf6792bd3e3ab4be9d6c5c"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Vansa",
   "axes": {
    "calibration": 86.52,
    "cost": 61.92,
    "intelligence": 40.85,
    "speed": 95.05
   },
   "calibration": {
    "score": 86.52
   },
   "capability": 63.68,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     25.3,
     68.97
    ],
    "B": [
     21.98,
     64.42
    ],
    "C": [
     17.57,
     49.9
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.034/M input, 0/M output (spec vansa-3.4.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.018590751269035533
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Vansa-3.4 (Vansa, hosted System One API)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 43.36,
    "I_sealed": 33.19,
    "base": 40.85,
    "per_type_split": {
     "open|choice": {
      "cc": 62.8658,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.4324,
       "hard": 56.5901,
       "judge": 64.4034,
       "standard": 62.3273
      }
     },
     "open|noul": {
      "cc": 32.7674,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.1429,
       "hard": 56.5217,
       "judge": 0.0,
       "standard": 22.2222
      }
     },
     "open|score": {
      "cc": 34.447,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.8152,
       "hard": 36.7794,
       "judge": 37.1156,
       "standard": 15.5952
      }
     },
     "sealed|choice": {
      "cc": 60.2137,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 66.6358,
       "hard": 45.7072,
       "judge": 63.8191,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 21.254,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 41.1765,
       "hard": 9.0909,
       "judge": 20.0,
       "standard": 37.5
      }
     },
     "sealed|score": {
      "cc": 18.1016,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 53.2589,
       "hard": -0.6743,
       "judge": 4.7068,
       "standard": 58.1668
      }
     }
    }
   },
   "jevbench_score": 42.57,
   "key": "vansa-3.4",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 27,
    "axes": {
     "calibration": 86.52,
     "cost": 61.92,
     "intelligence": 40.85,
     "speed": 95.49
    },
    "board_release": "v1.6.43",
    "capability": 63.68,
    "capability_api_board_rank": 23,
    "capability_open_board_rank": null,
    "capability_rank": 66,
    "composite_ci95": {
     "A": [
      25.32,
      69.03
     ],
     "B": [
      21.99,
      64.46
     ],
     "C": [
      17.58,
      49.94
     ]
    },
    "jevbench_score": 42.6,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 75,
    "ranks": {
     "A": 75,
     "B": 79,
     "C": 79,
     "capability": 108
    },
    "scores": {
     "A": 42.6,
     "B": 38.29,
     "C": 29.58
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.15,
     "p50_s_raw": 0.15,
     "p95_s_adjusted": 0.189,
     "p95_s_raw": 0.189
    },
    "views": null
   },
   "rank": 85,
   "ranked": true,
   "ranks": {
    "A": 85,
    "B": 90,
    "C": 91,
    "capability": 126
   },
   "repo": "https://vansa.org",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 42.57,
    "B": 38.27,
    "C": 29.56
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.158,
    "p50_s_raw": 0.158,
    "p95_s_adjusted": 0.198,
    "p95_s_raw": 0.198
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "84165e0500a8ebbc881b3cd1115354b060b09c1d964d82e41df5245a2cb22c6e"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 98.48,
    "cost": 38.98,
    "intelligence": 95.66,
    "speed": 71.82
   },
   "calibration": {
    "score": 98.48
   },
   "capability": 97.07,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     39.82,
     40.81
    ],
    "B": [
     42.21,
     43.68
    ],
    "C": [
     39.82,
     40.81
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.1/M input, 0.5/M output (spec gpt-6-luna-low.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.10814365482233503
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 3,
    "n_over_limit_answered_ok": 3,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "GPT-6 Luna (low reasoning effort)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 93.06,
    "I_sealed": 93.11,
    "base": 95.66,
    "per_type_split": {
     "open|choice": {
      "cc": 96.9832,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.0,
       "hard": 96.2252,
       "judge": 100.0,
       "standard": 92.4655
      }
     },
     "open|noul": {
      "cc": 96.3492,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 100.0,
       "judge": 100.0,
       "standard": 88.8889
      }
     },
     "open|score": {
      "cc": 85.8511,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 87.0909,
       "hard": 99.6667,
       "judge": 94.1525,
       "standard": 45.1481
      }
     },
     "sealed|choice": {
      "cc": 100.0,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100.0,
       "hard": 100.0,
       "judge": 100.0,
       "standard": 100.0
      }
     },
     "sealed|noul": {
      "cc": 92.1872,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 88.2353,
       "hard": 90.9091,
       "judge": 90.0,
       "standard": 100.0
      }
     },
     "sealed|score": {
      "cc": 87.1364,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 97.8611,
       "hard": 78.3133,
       "judge": 88.8182,
       "standard": 96.8976
      }
     }
    }
   },
   "jevbench_score": 40.39,
   "key": "gpt-6-luna-low",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "unknown",
   "previous": {
    "api_board_rank": 28,
    "axes": {
     "calibration": 99.02,
     "cost": 38.98,
     "intelligence": 96.62,
     "speed": 71.66
    },
    "board_release": "v1.6.43",
    "capability": 97.82,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      39.96,
      40.78
     ],
     "B": [
      42.4,
      43.66
     ],
     "C": [
      39.96,
      40.78
     ]
    },
    "jevbench_score": 40.48,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 81,
    "ranks": {
     "A": 81,
     "B": 70,
     "C": 64,
     "capability": 3
    },
    "scores": {
     "A": 40.48,
     "B": 43.16,
     "C": 40.48
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 600,
     "p50_s_adjusted": 1.854,
     "p50_s_raw": 1.854,
     "p95_s_adjusted": 3.683,
     "p95_s_raw": 3.683
    },
    "views": null
   },
   "rank": 93,
   "ranked": true,
   "ranks": {
    "A": 93,
    "B": 80,
    "C": 75,
    "capability": 2
   },
   "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna-low",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 40.39,
    "B": 43.02,
    "C": 40.39
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.896,
    "p50_s_raw": 1.896,
    "p95_s_adjusted": 3.465,
    "p95_s_raw": 3.465
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "c5fa6729c53fbe4fc0ec88af75ee26aa997274bc1ef4a820a8c468616c433e33"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Autoloops",
   "axes": {
    "calibration": 87.31,
    "cost": 40.47,
    "intelligence": 58.1,
    "speed": 84.44
   },
   "calibration": {
    "score": 87.31
   },
   "capability": 72.7,
   "class": "jev",
   "composite_ci95": {
    "A": [
     38.2,
     41.77
    ],
    "B": [
     37.03,
     42.17
    ],
    "C": [
     26.76,
     41.77
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.2/M input, 0.45/M output (spec kushal-gemma4-31b-it-autoloops.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.09648663282571913
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Autoloops – Gemma 4 31B IT",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 57.42,
    "I_sealed": 53.63,
    "base": 58.1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.1396,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.1081,
       "hard": 71.6892,
       "judge": 76.269,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 50.0193,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 47.8261,
       "judge": 40.0,
       "standard": 44.4444
      }
     },
     "open|score": {
      "cc": 47.1011,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.1922,
       "hard": 49.5059,
       "judge": 45.7484,
       "standard": 34.2749
      }
     },
     "sealed|choice": {
      "cc": 80.2748,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 88.8786,
       "hard": 78.6707,
       "judge": 81.9095,
       "standard": 76.7292
      }
     },
     "sealed|noul": {
      "cc": 39.9198,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 18.1818,
       "judge": 50.0,
       "standard": 50.0
      }
     },
     "sealed|score": {
      "cc": 40.6891,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 81.6366,
       "hard": 8.7267,
       "judge": 56.0954,
       "standard": 61.0306
      }
     }
    }
   },
   "jevbench_score": 40.17,
   "key": "kushal-gemma4-31b-it-autoloops",
   "last_measured_on": "2026-10-06",
   "licence": "Gemma terms of use (Google) for the base weights; hosted Autoloops service",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 29,
    "axes": {
     "calibration": 87.31,
     "cost": 40.47,
     "intelligence": 58.1,
     "speed": 84.51
    },
    "board_release": "v1.6.43",
    "capability": 72.7,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      38.21,
      41.78
     ],
     "B": [
      37.03,
      42.18
     ],
     "C": [
      26.77,
      41.78
     ]
    },
    "jevbench_score": 40.18,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 82,
    "ranks": {
     "A": 82,
     "B": 76,
     "C": 70,
     "capability": 58
    },
    "scores": {
     "A": 40.18,
     "B": 39.73,
     "C": 37.67
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.584,
     "p50_s_raw": 0.584,
     "p95_s_adjusted": 0.606,
     "p95_s_raw": 0.606
    },
    "views": null
   },
   "rank": 94,
   "ranked": true,
   "ranks": {
    "A": 94,
    "B": 87,
    "C": 78,
    "capability": 70
   },
   "repo": "https://benchmarkheaven.com/jev-models/kushal-gemma4-31b-it-autoloops",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 40.17,
    "B": 39.73,
    "C": 37.66
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.584,
    "p50_s_raw": 0.584,
    "p95_s_adjusted": 0.616,
    "p95_s_raw": 0.616
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "18ef06f9a56b5d3e358f5a8ddd0bf942facf1bae52fb8b57b5336b8026c1be5f"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 596,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 99.92,
    "cost": 38.41,
    "intelligence": 96.88,
    "speed": 71.38
   },
   "calibration": {
    "score": 99.92
   },
   "capability": 98.4,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     38.57,
     39.32
    ],
    "B": [
     40.95,
     42.13
    ],
    "C": [
     38.57,
     39.32
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.1/M input, 0.5/M output (spec gpt-6-luna.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.1129575296108291
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 5,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "GPT-6 Luna (default medium reasoning effort)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 95.37,
    "I_sealed": 93.25,
    "base": 96.88,
    "per_type_split": {
     "open|choice": {
      "cc": 99.245,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.0,
       "hard": 98.1126,
       "judge": 100.0,
       "standard": 100.0
      }
     },
     "open|noul": {
      "cc": 97.1429,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.4286,
       "hard": 100.0,
       "judge": 100.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 89.716,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 88.4545,
       "hard": 100.0,
       "judge": 93.5932,
       "standard": 63.963
      }
     },
     "sealed|choice": {
      "cc": 99.2244,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100.0,
       "hard": 98.061,
       "judge": 100.0,
       "standard": 100.0
      }
     },
     "sealed|noul": {
      "cc": 93.3636,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 100.0,
       "hard": 90.9091,
       "judge": 90.0,
       "standard": 100.0
      }
     },
     "sealed|score": {
      "cc": 87.1594,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 98.5972,
       "hard": 78.3133,
       "judge": 89.5,
       "standard": 95.622
      }
     }
    }
   },
   "jevbench_score": 39.1,
   "key": "gpt-6-luna",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "unknown",
   "previous": {
    "api_board_rank": 30,
    "axes": {
     "calibration": 100,
     "cost": 38.41,
     "intelligence": 96.88,
     "speed": 71.77
    },
    "board_release": "v1.6.43",
    "capability": 98.44,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      38.68,
      39.37
     ],
     "B": [
      41.06,
      42.18
     ],
     "C": [
      38.68,
      39.37
     ]
    },
    "jevbench_score": 39.16,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 84,
    "ranks": {
     "A": 84,
     "B": 72,
     "C": 66,
     "capability": 1
    },
    "scores": {
     "A": 39.16,
     "B": 41.79,
     "C": 39.16
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 596,
     "p50_s_adjusted": 1.824,
     "p50_s_raw": 1.824,
     "p95_s_adjusted": 3.646,
     "p95_s_raw": 3.646
    },
    "views": null
   },
   "rank": 96,
   "ranked": true,
   "ranks": {
    "A": 96,
    "B": 83,
    "C": 76,
    "capability": 1
   },
   "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 39.1,
    "B": 41.74,
    "C": 39.1
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.932,
    "p50_s_raw": 1.932,
    "p95_s_adjusted": 3.765,
    "p95_s_raw": 3.765
   },
   "status": {
    "answered_ok": 596,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "36dce9d9c3ec86a404450d2d6b5e9d6f57a229474e17c5a78a5805689d481088"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Fastino",
   "axes": {
    "calibration": 87.95,
    "cost": 35.86,
    "intelligence": 66.1,
    "speed": 77.23
   },
   "calibration": {
    "score": 87.95
   },
   "capability": 77.02,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     29.46,
     31.56
    ],
    "B": [
     29.6,
     32.79
    ],
    "C": [
     27.79,
     31.56
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.15/M input, 0/M output (spec fastino-glide.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.13735482233502538
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Fastino GLiDE",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 64.15,
    "I_sealed": 62.91,
    "base": 66.1,
    "per_type_split": {
     "open|choice": {
      "cc": 92.7738,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.0,
       "hard": 84.9009,
       "judge": 96.0448,
       "standard": 100.0
      }
     },
     "open|noul": {
      "cc": 34.3685,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 47.8261,
       "judge": 0.0,
       "standard": 33.3333
      }
     },
     "open|score": {
      "cc": 65.3016,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 78.1513,
       "hard": 64.5629,
       "judge": 75.2145,
       "standard": 45.485
      }
     },
     "sealed|choice": {
      "cc": 93.9895,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100.0,
       "hard": 88.3658,
       "judge": 95.4774,
       "standard": 100.0
      }
     },
     "sealed|noul": {
      "cc": 36.4305,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 52.9412,
       "hard": 9.0909,
       "judge": 50.0,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 58.3168,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 87.3406,
       "hard": 37.016,
       "judge": 63.7483,
       "standard": 78.2591
      }
     }
    }
   },
   "jevbench_score": 30.57,
   "key": "fastino-glide",
   "last_measured_on": "2026-10-06",
   "licence": "hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 32,
    "axes": {
     "calibration": 87.95,
     "cost": 35.86,
     "intelligence": 66.1,
     "speed": 77.48
    },
    "board_release": "v1.6.43",
    "capability": 77.02,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      29.48,
      31.58
     ],
     "B": [
      29.61,
      32.81
     ],
     "C": [
      27.81,
      31.58
     ]
    },
    "jevbench_score": 30.59,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 99,
    "ranks": {
     "A": 99,
     "B": 94,
     "C": 76,
     "capability": 39
    },
    "scores": {
     "A": 30.59,
     "B": 31.21,
     "C": 30.59
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.743,
     "p50_s_raw": 0.743,
     "p95_s_adjusted": 2.405,
     "p95_s_raw": 2.405
    },
    "views": null
   },
   "rank": 113,
   "ranked": true,
   "ranks": {
    "A": 113,
    "B": 107,
    "C": 88,
    "capability": 46
   },
   "repo": null,
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 30.57,
    "B": 31.2,
    "C": 30.57
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.852,
    "p50_s_raw": 0.852,
    "p95_s_adjusted": 2.218,
    "p95_s_raw": 2.218
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "a5736558e4af448fa482a03cbcbd4b119fc82c7695931ca18b671b86dfe1e724"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Featherless AI",
   "axes": {
    "calibration": 80.3,
    "cost": 35.15,
    "intelligence": 56.29,
    "speed": 76.09
   },
   "calibration": {
    "score": 80.3
   },
   "capability": 68.29,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     22.75,
     28.83
    ],
    "B": [
     22.25,
     29.49
    ],
    "C": [
     15.8,
     28.83
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 0.1451164454277286
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "SimpleJev Qwen3.6-35B-A3B",
   "endpoint_condition": "the author's public demo endpoint",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 60.15,
    "I_sealed": 47.27,
    "base": 56.29,
    "per_type_split": {
     "open|choice": {
      "cc": 73.3677,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.5405,
       "hard": 62.2523,
       "judge": 88.1345,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 55.5473,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 65.2174,
       "judge": 40.0,
       "standard": 44.4444
      }
     },
     "open|score": {
      "cc": 51.55,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 55.6168,
       "hard": 57.237,
       "judge": 65.3065,
       "standard": 17.5079
      }
     },
     "sealed|choice": {
      "cc": 60.9118,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 83.3179,
       "hard": 41.8292,
       "judge": 68.3417,
       "standard": 76.7292
      }
     },
     "sealed|noul": {
      "cc": 40.5963,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 88.2353,
       "hard": 18.1818,
       "judge": 40.0,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 40.3111,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 59.2125,
       "hard": 8.1459,
       "judge": 57.1294,
       "standard": 69.9633
      }
     }
    }
   },
   "jevbench_score": 27.53,
   "key": "simplejev-qwen3.6-35b-a3b",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 (Qwen weights); repository licence not stated",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 33,
    "axes": {
     "calibration": 80.3,
     "cost": 35.15,
     "intelligence": 56.29,
     "speed": 76.7
    },
    "board_release": "v1.6.43",
    "capability": 68.29,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      22.78,
      28.87
     ],
     "B": [
      22.27,
      29.53
     ],
     "C": [
      15.82,
      28.87
     ]
    },
    "jevbench_score": 27.57,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 106,
    "ranks": {
     "A": 106,
     "B": 100,
     "C": 95,
     "capability": 88
    },
    "scores": {
     "A": 27.57,
     "B": 27.61,
     "C": 24.26
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 591,
     "p50_s_adjusted": 1.25,
     "p50_s_raw": 0.625,
     "p95_s_adjusted": 1.709,
     "p95_s_raw": 0.8545
    },
    "views": null
   },
   "rank": 120,
   "ranked": true,
   "ranks": {
    "A": 120,
    "B": 114,
    "C": 108,
    "capability": 103
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 27.53,
    "B": 27.58,
    "C": 24.22
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.304,
    "p50_s_raw": 0.652,
    "p95_s_adjusted": 1.885,
    "p95_s_raw": 0.943
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "7cd32be5f5c8177492a086c471436c956ff89527c8b26c3705c4e939597ffd6e"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 94.31,
    "cost": 30.16,
    "intelligence": 92.97,
    "speed": 73.21
   },
   "calibration": {
    "score": 94.31
   },
   "capability": 93.64,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     20.98,
     21.75
    ],
    "B": [
     22.55,
     23.65
    ],
    "C": [
     20.98,
     21.75
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.2/M input, 1.2/M output (spec gpt-5.6-luna.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.2127451776649746
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 4,
    "n_over_limit_answered_ok": 4,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "GPT-5.6 Luna (low reasoning effort)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 91.05,
    "I_sealed": 89.75,
    "base": 92.97,
    "per_type_split": {
     "open|choice": {
      "cc": 93.8551,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.0,
       "hard": 94.3378,
       "judge": 92.0897,
       "standard": 92.4655
      }
     },
     "open|noul": {
      "cc": 100.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 100.0,
       "judge": 100.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 79.3051,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 77.1818,
       "hard": 94.0625,
       "judge": 83.7819,
       "standard": 44.137
      }
     },
     "sealed|choice": {
      "cc": 89.5553,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 94.4393,
       "hard": 82.5487,
       "judge": 95.4774,
       "standard": 92.2431
      }
     },
     "sealed|noul": {
      "cc": 94.6471,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 100.0,
       "judge": 90.0,
       "standard": 100.0
      }
     },
     "sealed|score": {
      "cc": 85.0542,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 90.2472,
       "hard": 78.3022,
       "judge": 91.2727,
       "standard": 86.6341
      }
     }
    }
   },
   "jevbench_score": 21.36,
   "key": "gpt-5.6-luna",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 34,
    "axes": {
     "calibration": 94.91,
     "cost": 30.16,
     "intelligence": 94.07,
     "speed": 73.12
    },
    "board_release": "v1.6.43",
    "capability": 94.49,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      21.08,
      21.77
     ],
     "B": [
      22.67,
      23.67
     ],
     "C": [
      21.08,
      21.77
     ]
    },
    "jevbench_score": 21.41,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 121,
    "ranks": {
     "A": 121,
     "B": 111,
     "C": 103,
     "capability": 5
    },
    "scores": {
     "A": 21.41,
     "B": 23.14,
     "C": 21.41
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 600,
     "p50_s_adjusted": 1.588,
     "p50_s_raw": 1.588,
     "p95_s_adjusted": 3.067,
     "p95_s_raw": 3.067
    },
    "views": null
   },
   "rank": 137,
   "ranked": true,
   "ranks": {
    "A": 137,
    "B": 126,
    "C": 118,
    "capability": 3
   },
   "repo": "https://benchmarkheaven.com/jev-models/gpt-5.6-luna",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.36,
    "B": 23.06,
    "C": 21.36
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.586,
    "p50_s_raw": 1.586,
    "p95_s_adjusted": 3.011,
    "p95_s_raw": 3.011
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "5d37f643d12df8d42d02c44dc364058832ddc8298187ebc522f43050b9199cac"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "ZooWork / pierre-srp",
   "axes": {
    "calibration": 86.08,
    "cost": 75.03,
    "intelligence": 28.34,
    "speed": 88.56
   },
   "calibration": {
    "score": 86.08
   },
   "capability": 57.21,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     10.92,
     37.03
    ],
    "B": [
     8.89,
     32.67
    ],
    "C": [
     7.58,
     25.72
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.01/M input, 0/M output (spec instinct-dual-4b.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.006799306260575296
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Instinct Dual 4B",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 26.29,
    "I_sealed": 25.26,
    "base": 28.34,
    "per_type_split": {
     "open|choice": {
      "cc": 51.7127,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.8649,
       "hard": 45.2658,
       "judge": 52.5379,
       "standard": 54.7928
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.8571,
       "hard": -4.3478,
       "judge": -10.0,
       "standard": -11.1111
      }
     },
     "open|score": {
      "cc": 27.145,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.8556,
       "hard": 24.9911,
       "judge": 27.0981,
       "standard": 15.6677
      }
     },
     "sealed|choice": {
      "cc": 49.0204,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 55.5144,
       "hard": 43.7682,
       "judge": 45.7286,
       "standard": 61.2153
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 17.6471,
       "hard": 9.0909,
       "judge": -30.0,
       "standard": 0.0
      }
     },
     "sealed|score": {
      "cc": 26.7459,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 42.7071,
       "hard": 24.586,
       "judge": 5.175,
       "standard": 55.4417
      }
     }
    }
   },
   "jevbench_score": 17.97,
   "key": "instinct-dual-4b",
   "last_measured_on": "2026-10-06",
   "licence": "Not independently recorded",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 36,
    "axes": {
     "calibration": 86.08,
     "cost": 75.03,
     "intelligence": 28.34,
     "speed": 88.6
    },
    "board_release": "v1.6.43",
    "capability": 57.21,
    "capability_api_board_rank": 25,
    "capability_open_board_rank": null,
    "capability_rank": 93,
    "composite_ci95": {
     "A": [
      10.92,
      37.04
     ],
     "B": [
      8.89,
      32.67
     ],
     "C": [
      7.58,
      25.72
     ]
    },
    "jevbench_score": 17.97,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 142,
    "ranks": {
     "A": 142,
     "B": 150,
     "C": 146,
     "capability": 147
    },
    "scores": {
     "A": 17.97,
     "B": 15.04,
     "C": 12.48
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 600,
     "p50_s_adjusted": 0.231,
     "p50_s_raw": 0.231,
     "p95_s_adjusted": 0.598,
     "p95_s_raw": 0.598
    },
    "views": null
   },
   "rank": 161,
   "ranked": true,
   "ranks": {
    "A": 161,
    "B": 169,
    "C": 165,
    "capability": 164
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/123",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.97,
    "B": 15.04,
    "C": 12.48
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.232,
    "p50_s_raw": 0.232,
    "p95_s_adjusted": 0.601,
    "p95_s_raw": 0.601
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "8d8a87d185d8fb317472c636b36cbb1f0ac8f03abef8c751bdfa290a9aacc746"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 599,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Google",
   "axes": {
    "calibration": 60.93,
    "cost": 29.17,
    "intelligence": 58.57,
    "speed": 78.98
   },
   "calibration": {
    "score": 60.93
   },
   "capability": 59.75,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     15.19,
     17.76
    ],
    "B": [
     15.24,
     18.64
    ],
    "C": [
     10.55,
     17.76
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.25/M input, 1.5/M output (spec gemini-3.1-flash-lite.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.22953722504230117
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Gemini 3.1 Flash-Lite",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 63.88,
    "I_sealed": 48.12,
    "base": 58.57,
    "per_type_split": {
     "open|choice": {
      "cc": 70.9158,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.3243,
       "hard": 73.5766,
       "judge": 72.3138,
       "standard": 54.7928
      }
     },
     "open|noul": {
      "cc": 62.9917,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 65.2174,
       "judge": 50.0,
       "standard": 66.6667
      }
     },
     "open|score": {
      "cc": 57.7334,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 43.1818,
       "hard": 58.3333,
       "judge": 56.0508,
       "standard": 66.3333
      }
     },
     "sealed|choice": {
      "cc": 66.1467,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 83.3179,
       "hard": 49.5853,
       "judge": 72.8643,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 56.3289,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 45.4545,
       "judge": 60.0,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 21.8863,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 54.2222,
       "hard": 21.553,
       "judge": -22.9545,
       "standard": 73.6463
      }
     }
    }
   },
   "jevbench_score": 16.93,
   "key": "gemini-3.1-flash-lite",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 37,
    "axes": {
     "calibration": 60.93,
     "cost": 29.17,
     "intelligence": 58.57,
     "speed": 79.03
    },
    "board_release": "v1.6.43",
    "capability": 59.75,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      15.19,
      17.77
     ],
     "B": [
      15.24,
      18.65
     ],
     "C": [
      10.55,
      17.77
     ]
    },
    "jevbench_score": 16.94,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 148,
    "ranks": {
     "A": 148,
     "B": 139,
     "C": 132,
     "capability": 134
    },
    "scores": {
     "A": 16.94,
     "B": 17.46,
     "C": 16.14
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 599,
     "p50_s_adjusted": 0.957,
     "p50_s_raw": 0.957,
     "p95_s_adjusted": 1.306,
     "p95_s_raw": 1.306
    },
    "views": null
   },
   "rank": 167,
   "ranked": true,
   "ranks": {
    "A": 167,
    "B": 157,
    "C": 151,
    "capability": 151
   },
   "repo": "https://benchmarkheaven.com/jev-models/gemini-3.1-flash-lite",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.93,
    "B": 17.46,
    "C": 16.14
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.962,
    "p50_s_raw": 0.962,
    "p95_s_adjusted": 1.317,
    "p95_s_raw": 1.317
   },
   "status": {
    "answered_ok": 599,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "96adf8183eaf9826b99e72a821469fa1fcc61783bc7b04ec2a95f0abcea883e8"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "mithalouni",
   "axes": {
    "calibration": 75.94,
    "cost": 68.33,
    "intelligence": 27.94,
    "speed": 79.18
   },
   "calibration": {
    "score": 75.94
   },
   "capability": 51.94,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     7.72,
     36.08
    ],
    "B": [
     6.26,
     32.42
    ],
    "C": [
     5.36,
     25.06
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "kind": "estimate",
    "usd_per_1000": 0.011367487684729064
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "system-one-open (Gemma 4 E2B LoRA on an L4)",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 27.92,
    "I_sealed": 22.82,
    "base": 27.94,
    "per_type_split": {
     "open|choice": {
      "cc": 56.8437,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2973,
       "hard": 47.1532,
       "judge": 72.3138,
       "standard": 54.7928
      }
     },
     "open|noul": {
      "cc": 3.8026,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.8571,
       "hard": 4.3478,
       "judge": 0.0,
       "standard": -11.1111
      }
     },
     "open|score": {
      "cc": 23.1119,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.1386,
       "hard": 15.9484,
       "judge": 27.2824,
       "standard": 18.1699
      }
     },
     "sealed|choice": {
      "cc": 47.7778,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 72.1965,
       "hard": 39.8901,
       "judge": 59.2965,
       "standard": 34.0659
      }
     },
     "sealed|noul": {
      "cc": 9.754,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 41.1765,
       "hard": 9.0909,
       "judge": -10.0,
       "standard": 25.0
      }
     },
     "sealed|score": {
      "cc": 10.9164,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 11.5419,
       "hard": 7.5012,
       "judge": -9.576,
       "standard": 48.1724
      }
     }
    }
   },
   "jevbench_score": 16.39,
   "key": "system-one-open",
   "last_measured_on": "2026-10-06",
   "licence": "MIT (repository LICENSE; Gemma weights keep Google’s terms)",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 38,
    "axes": {
     "calibration": 75.94,
     "cost": 68.33,
     "intelligence": 27.94,
     "speed": 79.32
    },
    "board_release": "v1.6.43",
    "capability": 51.94,
    "capability_api_board_rank": 26,
    "capability_open_board_rank": null,
    "capability_rank": 116,
    "composite_ci95": {
     "A": [
      7.72,
      36.09
     ],
     "B": [
      6.26,
      32.43
     ],
     "C": [
      5.36,
      25.06
     ]
    },
    "jevbench_score": 16.39,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 152,
    "ranks": {
     "A": 152,
     "B": 154,
     "C": 152,
     "capability": 171
    },
    "scores": {
     "A": 16.39,
     "B": 13.94,
     "C": 11.38
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 600,
     "p50_s_adjusted": 1.02,
     "p50_s_raw": 0.51,
     "p95_s_adjusted": 1.146,
     "p95_s_raw": 0.573
    },
    "views": null
   },
   "rank": 171,
   "ranked": true,
   "ranks": {
    "A": 171,
    "B": 172,
    "C": 172,
    "capability": 192
   },
   "repo": "https://github.com/mithalouni/system-one-open",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.39,
    "B": 13.94,
    "C": 11.38
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.028,
    "p50_s_raw": 0.514,
    "p95_s_adjusted": 1.174,
    "p95_s_raw": 0.587
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "1059a8be40a52bc4482c5bd40a45a368b53c2849acb4c370bc8735079c33e5a3"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "ekzhang",
   "axes": {
    "calibration": 78.57,
    "cost": 35.63,
    "intelligence": 38.77,
    "speed": 77.66
   },
   "calibration": {
    "score": 78.57
   },
   "capability": 58.67,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     8.74,
     26.17
    ],
    "B": [
     7.93,
     25.59
    ],
    "C": [
     6.07,
     18.17
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 0.13981028325123152
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "openjev-sglang (Qwen3.6-35B-A3B on SGLang)",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 44.34,
    "I_sealed": 28.06,
    "base": 38.77,
    "per_type_split": {
     "open|choice": {
      "cc": 69.7112,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.3243,
       "hard": 50.928,
       "judge": 68.3586,
       "standard": 100.0
      }
     },
     "open|noul": {
      "cc": 32.4458,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 39.1304,
       "judge": 20.0,
       "standard": 11.1111
      }
     },
     "open|score": {
      "cc": 30.8736,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.725,
       "hard": 32.6506,
       "judge": 31.5762,
       "standard": 20.8401
      }
     },
     "sealed|choice": {
      "cc": 56.672,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 77.7572,
       "hard": 37.9511,
       "judge": 63.8191,
       "standard": 72.8507
      }
     },
     "sealed|noul": {
      "cc": 2.9813,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 41.1765,
       "hard": -9.0909,
       "judge": 0.0,
       "standard": 12.5
      }
     },
     "sealed|score": {
      "cc": 24.5338,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 60.2254,
       "hard": 0.9111,
       "judge": 15.1583,
       "standard": 67.9965
      }
     }
    }
   },
   "jevbench_score": 15.38,
   "key": "openjev-sglang",
   "last_measured_on": "2026-10-06",
   "licence": "no licence file in the repository as of 2026-09-19; Qwen3.6 weights keep their own terms",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 39,
    "axes": {
     "calibration": 78.57,
     "cost": 35.63,
     "intelligence": 38.77,
     "speed": 77.71
    },
    "board_release": "v1.6.43",
    "capability": 58.67,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      8.74,
      26.17
     ],
     "B": [
      7.94,
      25.59
     ],
     "C": [
      6.07,
      18.18
     ]
    },
    "jevbench_score": 15.38,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 160,
    "ranks": {
     "A": 160,
     "B": 151,
     "C": 161,
     "capability": 139
    },
    "scores": {
     "A": 15.38,
     "B": 14.51,
     "C": 10.68
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 591,
     "p50_s_adjusted": 1.265,
     "p50_s_raw": 0.6325,
     "p95_s_adjusted": 1.339,
     "p95_s_raw": 0.6695
    },
    "views": null
   },
   "rank": 177,
   "ranked": true,
   "ranks": {
    "A": 177,
    "B": 170,
    "C": 179,
    "capability": 157
   },
   "repo": "https://github.com/ekzhang/openjev-sglang",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.38,
    "B": 14.51,
    "C": 10.68
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.27,
    "p50_s_raw": 0.635,
    "p95_s_adjusted": 1.35,
    "p95_s_raw": 0.675
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "16b2211d50e366983ff0094cc9cd19139fb0bdee7907d67f0240d1f03f900217"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 595,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "DeepSeek",
   "axes": {
    "calibration": 94.76,
    "cost": 19.56,
    "intelligence": 84.48,
    "speed": 65.99
   },
   "calibration": {
    "score": 94.76
   },
   "capability": 89.62,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     6.79,
     7.03
    ],
    "B": [
     7.41,
     7.83
    ],
    "C": [
     6.79,
     7.03
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.15/M input, 0.6/M output (spec deepseek-flash.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.480105076142132
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 34,
    "n_over_limit_answered_ok": 31,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "DeepSeek V4.1 Flash (thinking default)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 82.33,
    "I_sealed": 81.49,
    "base": 84.48,
    "per_type_split": {
     "open|choice": {
      "cc": 95.4703,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.0,
       "hard": 88.6757,
       "judge": 100.0,
       "standard": 100.0
      }
     },
     "open|noul": {
      "cc": 93.5217,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.3043,
       "judge": 90.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 57.9899,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.9091,
       "hard": 71.9722,
       "judge": 68.4237,
       "standard": 22.4148
      }
     },
     "sealed|choice": {
      "cc": 96.1218,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100.0,
       "hard": 94.1829,
       "judge": 100.0,
       "standard": 92.2431
      }
     },
     "sealed|noul": {
      "cc": 87.8636,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 100.0,
       "hard": 90.9091,
       "judge": 80.0,
       "standard": 87.5
      }
     },
     "sealed|score": {
      "cc": 60.4774,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 38.6944,
       "hard": 40.4217,
       "judge": 94.5455,
       "standard": 60.378
      }
     }
    }
   },
   "jevbench_score": 6.9,
   "key": "deepseek-flash",
   "last_measured_on": "2026-10-06",
   "licence": "open weights, proprietary API route",
   "listing": "ranked",
   "method": "v1.7",
   "open": "weights",
   "previous": {
    "api_board_rank": 40,
    "axes": {
     "calibration": 100,
     "cost": 19.56,
     "intelligence": 94.71,
     "speed": 68.07
    },
    "board_release": "v1.6.43",
    "capability": 97.35,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      7.02,
      7.13
     ],
     "B": [
      7.78,
      7.99
     ],
     "C": [
      7.02,
      7.13
     ]
    },
    "jevbench_score": 7.09,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 180,
    "ranks": {
     "A": 180,
     "B": 174,
     "C": 172,
     "capability": 4
    },
    "scores": {
     "A": 7.09,
     "B": 7.89,
     "C": 7.09
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 595,
     "p50_s_adjusted": 2.038,
     "p50_s_raw": 2.038,
     "p95_s_adjusted": 7.648,
     "p95_s_raw": 7.648
    },
    "views": null
   },
   "rank": 202,
   "ranked": true,
   "ranks": {
    "A": 202,
    "B": 197,
    "C": 194,
    "capability": 4
   },
   "repo": "https://benchmarkheaven.com/jev-models/deepseek-flash",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 6.9,
    "B": 7.61,
    "C": 6.9
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 2.205,
    "p50_s_raw": 2.205,
    "p95_s_adjusted": 11.414,
    "p95_s_raw": 11.414
   },
   "status": {
    "answered_ok": 595,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "c9078842b58b17d1e14e86c940cbe4291066ec8a620c430aac3b4385e656feb0"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Featherless AI",
   "axes": {
    "calibration": 90.33,
    "cost": 14.9,
    "intelligence": 52.88,
    "speed": 76.44
   },
   "calibration": {
    "score": 90.33
   },
   "capability": 71.6,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     2.1,
     3.31
    ],
    "B": [
     2.18,
     3.6
    ],
    "C": [
     1.46,
     3.31
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 0.686803448275862
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "SimpleJev Qwen3.8-27B",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 58.03,
    "I_sealed": 42.58,
    "base": 52.88,
    "per_type_split": {
     "open|choice": {
      "cc": 77.1205,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.2162,
       "hard": 67.9144,
       "judge": 84.1793,
       "standard": 77.3964
      }
     },
     "open|noul": {
      "cc": 39.7633,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 39.1304,
       "judge": 10.0,
       "standard": 55.5556
      }
     },
     "open|score": {
      "cc": 57.2101,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.6453,
       "hard": 49.2044,
       "judge": 65.6588,
       "standard": 51.3309
      }
     },
     "sealed|choice": {
      "cc": 61.5454,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 72.1965,
       "hard": 47.6462,
       "judge": 63.8191,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 31.4198,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 18.1818,
       "judge": 30.0,
       "standard": 37.5
      }
     },
     "sealed|score": {
      "cc": 34.7637,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 72.2528,
       "hard": 8.1052,
       "judge": 33.119,
       "standard": 71.8031
      }
     }
    }
   },
   "jevbench_score": 3.22,
   "key": "simplejev-qwen3.8-27b",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 (Qwen weights); repository licence not stated",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 41,
    "axes": {
     "calibration": 90.33,
     "cost": 14.9,
     "intelligence": 52.88,
     "speed": 76.93
    },
    "board_release": "v1.6.43",
    "capability": 71.6,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      2.11,
      3.31
     ],
     "B": [
      2.18,
      3.6
     ],
     "C": [
      1.46,
      3.31
     ]
    },
    "jevbench_score": 3.22,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 193,
    "ranks": {
     "A": 193,
     "B": 186,
     "C": 190,
     "capability": 66
    },
    "scores": {
     "A": 3.22,
     "B": 3.44,
     "C": 2.5
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 591,
     "p50_s_adjusted": 1.212,
     "p50_s_raw": 0.606,
     "p95_s_adjusted": 1.675,
     "p95_s_raw": 0.8375
    },
    "views": null
   },
   "rank": 215,
   "ranked": true,
   "ranks": {
    "A": 215,
    "B": 209,
    "C": 212,
    "capability": 76
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.22,
    "B": 3.44,
    "C": 2.5
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.266,
    "p50_s_raw": 0.633,
    "p95_s_adjusted": 1.794,
    "p95_s_raw": 0.897
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "f12f4c57ca739705117c34ceb3f847b59b15f9744120c83faf519a83ba9c123b"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "milliseconds.ai (Baptiste Laget)",
   "axes": {
    "calibration": 79.15,
    "cost": 58.33,
    "intelligence": 13.11,
    "speed": 92.87
   },
   "calibration": {
    "score": 79.15
   },
   "capability": 46.13,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     0.89,
     8.58
    ],
    "B": [
     0.64,
     7.01
    ],
    "C": [
     0.62,
     5.96
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.04/M input, 0/M output (spec decision-machine-1.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.02449299492385787
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "decision-machine-1 (milliseconds.ai)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 10.71,
    "I_sealed": 10.36,
    "base": 13.11,
    "per_type_split": {
     "open|choice": {
      "cc": 32.1365,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.8378,
       "hard": 13.1802,
       "judge": 36.7172,
       "standard": 62.3273
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.2857,
       "hard": -47.8261,
       "judge": -50.0,
       "standard": -66.6667
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 37.3934,
       "hard": -8.8505,
       "judge": 13.5155,
       "standard": -60.8938
      }
     },
     "sealed|choice": {
      "cc": 31.0657,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 38.8323,
       "hard": 34.073,
       "judge": 27.6382,
       "standard": 26.309
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 5.8824,
       "hard": -72.7273,
       "judge": -90.0,
       "standard": 0.0
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 8.9152,
       "hard": -27.1485,
       "judge": -46.9134,
       "standard": 35.2328
      }
     }
    }
   },
   "jevbench_score": 2.35,
   "key": "decision-machine-1",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API, closed weights",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 42,
    "axes": {
     "calibration": 79.15,
     "cost": 58.33,
     "intelligence": 13.11,
     "speed": 92.97
    },
    "board_release": "v1.6.43",
    "capability": 46.13,
    "capability_api_board_rank": 27,
    "capability_open_board_rank": null,
    "capability_rank": 126,
    "composite_ci95": {
     "A": [
      0.89,
      8.58
     ],
     "B": [
      0.64,
      7.01
     ],
     "C": [
      0.62,
      5.96
     ]
    },
    "jevbench_score": 2.35,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 196,
    "ranks": {
     "A": 196,
     "B": 196,
     "C": 196,
     "capability": 185
    },
    "scores": {
     "A": 2.35,
     "B": 1.78,
     "C": 1.63
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.167,
     "p50_s_raw": 0.167,
     "p95_s_adjusted": 0.303,
     "p95_s_raw": 0.303
    },
    "views": null
   },
   "rank": 219,
   "ranked": true,
   "ranks": {
    "A": 219,
    "B": 219,
    "C": 219,
    "capability": 207
   },
   "repo": "https://www.milliseconds.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 2.35,
    "B": 1.78,
    "C": 1.63
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.171,
    "p50_s_raw": 0.171,
    "p95_s_adjusted": 0.302,
    "p95_s_raw": 0.302
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "9fb20f7a1415fa931bb212eade8b88af31affff7800a8f45949b8b519e3bf2be"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 579,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "einptein",
   "axes": {
    "calibration": 47.44,
    "cost": 68.56,
    "intelligence": 10.1,
    "speed": 76.36
   },
   "calibration": {
    "score": 47.44
   },
   "capability": 28.77,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.3,
     5.7
    ],
    "B": [
     0.21,
     4.67
    ],
    "C": [
     0.21,
     3.96
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "kind": "estimate",
    "usd_per_1000": 0.011167487684729064
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "JevAct (einptein, jev1-2b-v2)",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 7.46,
    "I_sealed": 7.6,
    "base": 10.1,
    "per_type_split": {
     "open|choice": {
      "cc": 22.3825,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.9459,
       "hard": 9.4055,
       "judge": 40.6724,
       "standard": 17.1201
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0.0,
       "hard": -30.4348,
       "judge": -40.0,
       "standard": -33.3333
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 34.0168,
       "hard": -40.2474,
       "judge": 11.2946,
       "standard": -47.936
      }
     },
     "sealed|choice": {
      "cc": 22.8059,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 22.1501,
       "hard": 22.4389,
       "judge": 18.593,
       "standard": 30.1875
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 29.4118,
       "hard": 9.0909,
       "judge": -10.0,
       "standard": -25.0
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 2.6763,
       "hard": -73.6675,
       "judge": -35.3444,
       "standard": -0.6115
      }
     }
    }
   },
   "jevbench_score": 1.11,
   "key": "jevact",
   "last_measured_on": "2026-10-06",
   "licence": "not stated (the author published only an endpoint and the evaluation client)",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 43,
    "axes": {
     "calibration": 47.44,
     "cost": 68.56,
     "intelligence": 10.1,
     "speed": 76.45
    },
    "board_release": "v1.6.43",
    "capability": 28.77,
    "capability_api_board_rank": 29,
    "capability_open_board_rank": null,
    "capability_rank": 160,
    "composite_ci95": {
     "A": [
      0.3,
      5.7
     ],
     "B": [
      0.21,
      4.67
     ],
     "C": [
      0.21,
      3.96
     ]
    },
    "jevbench_score": 1.11,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 198,
    "ranks": {
     "A": 198,
     "B": 198,
     "C": 198,
     "capability": 231
    },
    "scores": {
     "A": 1.11,
     "B": 0.83,
     "C": 0.77
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 579,
     "p50_s_adjusted": 0.801,
     "p50_s_raw": 0.4005,
     "p95_s_adjusted": 2.829,
     "p95_s_raw": 1.4145
    },
    "views": null
   },
   "rank": 221,
   "ranked": true,
   "ranks": {
    "A": 221,
    "B": 221,
    "C": 221,
    "capability": 253
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/66",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 1.11,
    "B": 0.83,
    "C": 0.77
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.808,
    "p50_s_raw": 0.404,
    "p95_s_adjusted": 2.861,
    "p95_s_raw": 1.431
   },
   "status": {
    "answered_ok": 579,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "67145bec630ccceb1b6cbf2ed3545a5fd7ece3aa60a4e43e06f395b75da7c9f5"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 592,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Qwen / Chutes",
   "axes": {
    "calibration": 63.81,
    "cost": 0,
    "intelligence": 33.36,
    "speed": 54.99
   },
   "calibration": {
    "score": 63.81
   },
   "capability": 48.59,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     0.0,
     0.0
    ],
    "B": [
     0.0,
     0.0
    ],
    "C": [
     0.0,
     0.0
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 2.1783887438423646
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 233,
    "n_over_limit_answered_ok": 225,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Qwen3.8 27B (Chutes TEE)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 32.85,
    "I_sealed": 28.72,
    "base": 33.36,
    "per_type_split": {
     "open|choice": {
      "cc": 50.7411,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.3243,
       "hard": 24.5046,
       "judge": 60.4483,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 47.8226,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 21.7391,
       "judge": 50.0,
       "standard": 77.7778
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -46.0,
       "hard": -52.8611,
       "judge": -44.6102,
       "standard": -18.0
      }
     },
     "sealed|choice": {
      "cc": 46.11,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 55.5144,
       "hard": 30.195,
       "judge": 59.2965,
       "standard": 53.4583
      }
     },
     "sealed|noul": {
      "cc": 40.0561,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 27.2727,
       "judge": 30.0,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 24.5833,
       "hard": -130.1205,
       "judge": -150.0,
       "standard": -3.5
      }
     }
    }
   },
   "jevbench_score": 0.0,
   "key": "qwen3.8-27b",
   "last_measured_on": "2026-10-06",
   "licence": "open weights",
   "listing": "ranked",
   "method": "v1.7",
   "open": "weights",
   "previous": {
    "api_board_rank": 45,
    "axes": {
     "calibration": 100,
     "cost": 0,
     "intelligence": 96.38,
     "speed": 57.43
    },
    "board_release": "v1.6.43",
    "capability": 98.19,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0,
      0
     ],
     "B": [
      0,
      0
     ],
     "C": [
      0,
      0
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 242,
    "ranks": {
     "A": 242,
     "B": 242,
     "C": 242,
     "capability": 2
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 592,
     "p50_s_adjusted": 7.095,
     "p50_s_raw": 7.095,
     "p95_s_adjusted": 25.469,
     "p95_s_raw": 25.469
    },
    "views": null
   },
   "rank": 264,
   "ranked": true,
   "ranks": {
    "A": 264,
    "B": 264,
    "C": 264,
    "capability": 199
   },
   "repo": "https://benchmarkheaven.com/jev-models/qwen3.8-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.0,
    "B": 0.0,
    "C": 0.0
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 8.35,
    "p50_s_raw": 8.35,
    "p95_s_adjusted": 37.933,
    "p95_s_raw": 37.933
   },
   "status": {
    "answered_ok": 592,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "e513ccfd7f429c1bfc4533c40ba34037b9951812bbddb6d01db7e0ef9753386f"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 598,
    "round": "score-a5-1",
    "subset": "A5"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 90.1,
    "cost": 48.6,
    "intelligence": 56.9,
    "speed": 88.31
   },
   "calibration": {
    "score": 90.1
   },
   "capability": 73.5,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     51.52,
     64.44
    ],
    "B": [
     48.52,
     63.63
    ],
    "C": [
     35.78,
     64.44
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A5 u P run x tariff USD 0.1/M input, 0.0/M output (spec openai-decisions.json; OpenAI Decisions API (POST /v1/decisions, gpt-6-luna) list price USD 0.10 per 1M input tokens, no output/cache charges; source https://developers.openai.com/api/docs/guides/decisions (checked 2026-10-06); actual usage), per 1,000 decisions on the v1.6.1 common cost basis (592 of 600 items; the 8 A5 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a5.json). List price as of 6 Oct 2026 (launch day); re-checked at each revision.",
    "kind": "tariff",
    "usd_per_1000": 0.0517
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "OpenAI Decisions (gpt-6-luna)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 58.07,
    "I_sealed": 44.16,
    "base": 56.9,
    "per_type_split": {
     "open|choice": {
      "cc": 72.5513,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.1081,
       "hard": 62.2523,
       "judge": 80.2241,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 48.9124,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.1429,
       "hard": 65.2174,
       "judge": 20.0,
       "standard": 55.5556
      }
     },
     "open|score": {
      "cc": 52.7419,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.2727,
       "hard": 53.7315,
       "judge": 54.4068,
       "standard": 50.0
      }
     },
     "sealed|choice": {
      "cc": 69.2318,
      "n": {
       "easy": 17,
       "hard": 73,
       "judge": 38,
       "standard": 22
      },
      "tiers": {
       "easy": 75.8389,
       "hard": 64.8643,
       "judge": 76.453,
       "standard": 63.8312
      }
     },
     "sealed|noul": {
      "cc": 20.6086,
      "n": {
       "easy": 15,
       "hard": 17,
       "judge": 22,
       "standard": 21
      },
      "tiers": {
       "easy": 33.3333,
       "hard": 17.6471,
       "judge": 18.1818,
       "standard": 23.8095
      }
     },
     "sealed|score": {
      "cc": 42.6453,
      "n": {
       "easy": 9,
       "hard": 30,
       "judge": 21,
       "standard": 15
      },
      "tiers": {
       "easy": 55.64,
       "hard": 50.4536,
       "judge": 35.5753,
       "standard": 31.1364
      }
     }
    }
   },
   "jevbench_score": 62.37,
   "key": "openai-decisions",
   "last_measured_on": "2026-10-06",
   "licence": "hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 22,
    "axes": {
     "calibration": 90.1,
     "cost": 48.6,
     "intelligence": 56.9,
     "speed": 89.25
    },
    "board_release": "v1.6.43",
    "capability": 73.5,
    "capability_api_board_rank": 15,
    "capability_open_board_rank": null,
    "capability_rank": 29,
    "composite_ci95": {
     "A": [
      51.61,
      64.57
     ],
     "B": [
      48.59,
      63.73
     ],
     "C": [
      35.84,
      64.57
     ]
    },
    "jevbench_score": 62.5,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 53,
    "ranks": {
     "A": 53,
     "B": 51,
     "C": 46,
     "capability": 53
    },
    "scores": {
     "A": 62.5,
     "B": 60.53,
     "C": 56.2
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 598,
     "p50_s_adjusted": 0.298,
     "p50_s_raw": 0.298,
     "p95_s_adjusted": 0.399,
     "p95_s_raw": 0.399
    },
    "views": null
   },
   "rank": 60,
   "ranked": true,
   "ranks": {
    "A": 60,
    "B": 57,
    "C": 50,
    "capability": 64
   },
   "repo": null,
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 62.37,
    "B": 60.43,
    "C": 56.09
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.305,
    "p50_s_raw": 0.305,
    "p95_s_adjusted": 0.484,
    "p95_s_raw": 0.484
   },
   "status": {
    "answered_ok": 598,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A5",
    "equated": true,
    "equating_offset": {
     "C": 6.152420143549691,
     "I": 5.780609498995467
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a5-1",
    "run_sha256": "83c37a9258466d201a3a3fb30f5245bd2fa77f29f3c02b8594e68934a7bb77aa"
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 86.78599512136702,
    "cost": 62.84038569943876,
    "intelligence": 62.05738930369758,
    "speed": 87.8529201384932
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07532022247062888,
      "ece_score": 84.93595550587423,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 84.93595550587423,
      "score_before_time_limit": 84.93595550587423,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.17333333333333334,
      "brier": 0.0993520721969318,
      "ece": 0.07450471425946179,
      "n": 375,
      "score": 85.09905714810763,
      "score_before_time_limit": 85.09905714810763,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9386666666666666,
      "interval90_width": 0.28222222222222215,
      "n": 375,
      "nrps": 0.06855989887138964,
      "score": 90.32297271011922,
      "score_before_time_limit": 90.32297271011922,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.062490323463112944
     }
    },
    "score": 86.78599512136702
   },
   "capability": 74.4216922125323,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     70.39745141854905,
     74.05433680202529
    ],
    "B": [
     67.24590136651118,
     72.27360295159241
    ],
    "C": [
     63.53024774822211,
     74.05433680202529
    ]
   },
   "cost": {
    "basis": "Liquid AI d1 public API tariff USD 0.04 per 1M input tokens, no output tokens (Vercel AI Gateway listing https://vercel.com/ai-gateway/models/d1, read 6 Oct 2026; equals Liquid API usage.cost); measured on the v1.6.1 common cost basis from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.050374800000000004
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.017324224779959378
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "d1 (Liquid AI)",
   "endpoint_condition": "Liquid AI hosted API, text only, measured from Sandy (Helsinki) on 6 Oct 2026; full set 1,200 sealed + P300 (v1.6.1), not equated.",
   "endpoint_kind": "api",
   "full_api_addon": {
    "measured_on": "2026-10-06",
    "results_sha256": "e8af18a525745185563af558f24c97605cc4c5402d945dc9ce8ad5598ffe2606",
    "round": "a4-3"
   },
   "full_coverage": true,
   "intelligence": {
    "I_open": 64.87457995498468,
    "I_sealed": 59.240198652410484,
    "base": 62.05738930369758,
    "excess": 3.060993660135954,
    "gap": 5.634381302574198,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.4599081792597,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 71.68921044353571,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 61.42028985507247,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 57.74354183062191,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.53512949354776,
       "hard": 58.97474523537042,
       "judge": 58.364773735967425,
       "standard": 53.453493331643685
      }
     },
     "sealed|choice": {
      "cc": 71.1719233348168,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 78.71554208853745,
       "hard": 64.87556561085972,
       "judge": 71.13594040968344,
       "standard": 80.04680379357065
      }
     },
     "sealed|noul": {
      "cc": 46.350422823090824,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76.0,
       "hard": 22.330097087378633,
       "judge": 55.263157894736835,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 60.19824979932386,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 76.02006250272461,
       "hard": 54.94396966222077,
       "judge": 65.34461858109049,
       "standard": 55.07635054917971
      }
     }
    }
   },
   "jevbench_score": 72.81636405537589,
   "key": "liquid-d1",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "noul_decisive": {
    "acc_among_decisive": 0.9258064516129032,
    "decisive_rate": 0.8266666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 10,
    "axes": {
     "calibration": 86.78599512136702,
     "cost": 62.84038569943876,
     "intelligence": 62.05738930369758,
     "speed": 89.22497122158413
    },
    "board_release": "v1.6.43",
    "capability": 74.4216922125323,
    "capability_api_board_rank": 11,
    "capability_open_board_rank": null,
    "capability_rank": 25,
    "composite_ci95": {
     "A": [
      70.61498257469916,
      74.29509345786346
     ],
     "B": [
      67.40457796119317,
      72.4569260841226
     ],
     "C": [
      63.72660803369401,
      74.29509345786346
     ]
    },
    "jevbench_score": 73.0491258431614,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 13,
    "ranks": {
     "A": 13,
     "B": 19,
     "C": 13,
     "capability": 47
    },
    "scores": {
     "A": 73.0491258431614,
     "B": 70.54993499095869,
     "C": 73.0491258431614
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.26846902817487717,
     "p50_s_raw": 0.26846902817487717,
     "p95_s_adjusted": 0.44525487907230826,
     "p95_s_raw": 0.44525487907230826
    },
    "views": {
     "Balanced 33:33:33": 69.38810749276507,
     "Emphasis on Accuracy 60:20:20": 66.25736619060567,
     "Emphasis on Cost 20:20:60": 66.61183085641215,
     "Emphasis on Speed 20:60:20": 76.16108626295785,
     "Intelligence only": 62.05738930369758
    }
   },
   "rank": 16,
   "ranked": true,
   "ranks": {
    "A": 16,
    "B": 21,
    "C": 16,
    "capability": 58
   },
   "repo": "https://www.liquid.ai/blog/d1-decision-model",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.81636405537589,
    "B": 70.37612304785493,
    "C": 72.81636405537589
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26902734860777855,
    "p50_s_raw": 0.26902734860777855,
    "p95_s_adjusted": 0.609412775933742,
    "p95_s_raw": 0.609412775933742
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.98139688401669,
      88.21215531212913
     ],
     "I": [
      56.94190030827255,
      65.96370651107074
     ],
     "capability": [
      70.07219589958059,
      76.5079704884497
     ],
     "composite": {
      "A": [
       70.39745141854905,
       74.05433680202529
      ],
      "B": [
       67.24590136651118,
       72.27360295159241
      ],
      "C": [
       63.53024774822211,
       74.05433680202529
      ]
     },
     "seed": 115
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.78599512136702,
     "capability": 74.4216922125323,
     "intelligence": 62.05738930369758
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7866666666666666,
      "calibration": 84.93595550587423,
      "cc": 72.7177135928308,
      "cc_by_type": {
       "choice": 72.7177135928308
      },
      "cc_tiered": 73.31591575703825,
      "cc_tiered_open": 75.4599081792597,
      "cc_tiered_sealed": 71.1719233348168,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 72.7177135928308
     },
     "noul": {
      "accuracy": 0.7653333333333333,
      "calibration": 85.09905714810763,
      "cc": 53.06666666666666,
      "cc_by_type": {
       "noul": 53.06666666666666
      },
      "cc_tiered": 53.885356339081646,
      "cc_tiered_open": 61.42028985507247,
      "cc_tiered_sealed": 46.350422823090824,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.06666666666666
     },
     "score": {
      "accuracy": 0.7066666666666667,
      "calibration": 90.32297271011922,
      "cc": 59.75373554858271,
      "cc_by_type": {
       "score": 59.75373554858271
      },
      "cc_tiered": 58.970895814972884,
      "cc_tiered_open": 57.74354183062191,
      "cc_tiered_sealed": 60.19824979932386,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 59.75373554858271
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.10832417715342,
    "Emphasis on Accuracy 60:20:20": 66.1040387730098,
    "Emphasis on Cost 20:20:60": 66.45686042235283,
    "Emphasis on Speed 20:60:20": 75.5567408098811,
    "Intelligence only": 62.05738930369758
   }
  },
  {
   "api_flag": true,
   "author": "Microsoft",
   "axes": {
    "calibration": 84.39873071167106,
    "cost": 61.452075716621465,
    "intelligence": 57.16958510591244,
    "speed": 81.38546065308665
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09636174358194516,
      "ece_score": 80.72765128361097,
      "mean_tvd": null,
      "n": 735,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.72765128361097,
      "score_before_time_limit": 80.72765128361097,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2278820375335121,
      "brier": 0.10202697897728127,
      "ece": 0.05824671413738715,
      "n": 373,
      "score": 88.35065717252257,
      "score_before_time_limit": 88.35065717252257,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9056603773584906,
      "interval90_width": 0.21495956873315356,
      "n": 371,
      "nrps": 0.07658334569212163,
      "score": 84.11788367887968,
      "score_before_time_limit": 84.11788367887968,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.12052949036514232
     }
    },
    "score": 84.39873071167106
   },
   "capability": 70.78415790879176,
   "capability_ci95": [
    65.79924499904618,
    73.0808555865328
   ],
   "composite_ci95": {
    "A": [
     65.99814392729574,
     70.37976430138654
    ],
    "B": [
     62.310798268533894,
     68.07228955990982
    ],
    "C": [
     46.72503296526089,
     70.24484593257228
    ]
   },
   "cost": {
    "basis": "Advertised Azure Foundry tariff .042/M input, output free; genuine native usage, billed invoice unknown",
    "common_basis": "{\"n_items\": 1479, \"rule\": \"common\", \"usd_per_1000_all_items\": 0.019002368000000002}",
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0192721784989858
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Microsoft-Decision-1 (Azure Foundry)",
   "endpoint_kind": "api",
   "fresh_api_addendum": {
    "acceptance_sha256": "1a21b7d132e77af0196edebb1c4f9060e2eb3067aa04ec5ed99cb94bdbc808ce",
    "categories_sha256": "47a4db8923dc9edadcd74ee357c852099a7473e8f6b99100e8fe61767b9e63c5",
    "cost_mask_sha256": "50d582d3976ba79d458756f931764cf6e80bc623fbbe2fa611d8744752e3fc7b",
    "cost_n": 1479,
    "counts": {
     "P": 300,
     "S": 1200
    },
    "g_med_s": 7.088097744249485,
    "input_sha256": "6833e921ae525a328a4cf199a3e8145fa657fe2c9a9fe520e2953aab6da60ee6",
    "method": "O1S",
    "parent": "v1.6-fastlane-20261009",
    "release": "CR-401 Microsoft-Decision-1 full native API addendum",
    "release_authorization_sha256": "ea7e7d50d31ba80c47bdba27f9b32f35f03dabfd2bb1fe6264f6e9d5588023a3",
    "run_sha256": "cadf99e94de455277604759b31028cfb14663f3b815555e5ea7909e0bb891c27",
    "scorer_sha256": "83398383da9d011cc30236d0a241a9c125fc5efbbfc041f5564e7c5f699d896e",
    "source_url": "https://benchmarkheaven.com/api/jevbench/api-addenda"
   },
   "intelligence": {
    "I_open": 65.54484720922025,
    "I_sealed": 49.68435126148425,
    "base": 57.61459923535225,
    "excess": 8.772398203486514,
    "gap": 15.860495947735998,
    "penalty": 0.9922760179651349,
    "per_type_split": {
     "open|choice": {
      "cc": 82.42557001849185,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 75.46398238439761,
       "judge": 84.17930125247199,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 51.46376811594203,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 47.82608695652173,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 62.74520349322689,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.19290622964672,
       "hard": 67.62102226028924,
       "judge": 74.3930948447687,
       "standard": 41.29787756357959
      }
     },
     "sealed|choice": {
      "cc": 65.77933402862911,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 85.18366087045993,
       "hard": 59.4822734946539,
       "judge": 65.37585421412302,
       "standard": 69.27651139742319
      }
     },
     "sealed|noul": {
      "cc": 37.62824744344246,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 54.16666666666667,
       "hard": 25.86206896551724,
       "judge": 31.343283582089555,
       "standard": 62.318840579710155
      }
     },
     "sealed|score": {
      "cc": 45.6454723123812,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 68.67980276922125,
       "hard": 32.88780129611539,
       "judge": 46.813302889415894,
       "standard": 57.89190325094079
      }
     }
    }
   },
   "jevbench_score": 69.08416559129289,
   "key": "microsoft-decision-1",
   "last_measured_on": "2026-10-10",
   "listing": "ranked",
   "method": "v1.7",
   "model_pin": {
    "deployment": "decision-1",
    "model": "Microsoft-Decision-1",
    "provider": "Microsoft/Azure Foundry",
    "response_model": "microsoft-decision-1",
    "version": "1"
   },
   "noul_decisive": {
    "acc_among_decisive": 0.9340277777777778,
    "decisive_rate": 0.7721179624664879,
    "supported": true,
    "valid": 373
   },
   "previous": {
    "api_board_rank": 17,
    "axes": {
     "calibration": 84.39873071167106,
     "cost": 61.452075716621465,
     "intelligence": 57.16958510591244,
     "speed": 81.54517543213291
    },
    "board_release": "v1.6.43",
    "capability": 70.78415790879176,
    "capability_api_board_rank": 17,
    "capability_open_board_rank": null,
    "capability_rank": 41,
    "composite_ci95": {
     "A": [
      66.02436050159598,
      70.40957825472715
     ],
     "B": [
      62.32949160874876,
      68.09460025172818
     ],
     "C": [
      46.743685735358696,
      70.27466026550705
     ]
    },
    "jevbench_score": 69.11289175277149,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 29,
    "ranks": {
     "A": 29,
     "B": 38,
     "C": 40,
     "capability": 71
    },
    "scores": {
     "A": 69.11289175277149,
     "B": 66.34102888286164,
     "C": 62.74608663211473
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.45925578847527504,
     "p50_s_raw": 0.45925578847527504,
     "p95_s_adjusted": 1.5255548182874916,
     "p95_s_raw": 1.5255548182874916
    },
    "views": null
   },
   "rank": 34,
   "ranked": true,
   "ranks": {
    "A": 34,
    "B": 42,
    "C": 43,
    "capability": 84
   },
   "repo": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.08416559129289,
    "B": 66.31985230612224,
    "C": 62.72000677680797
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4787515588104725,
    "p50_s_raw": 0.4787515588104725,
    "p95_s_adjusted": 1.5182515516877173,
    "p95_s_raw": 1.5182515516877173
   },
   "status": {
    "answered_ok": 1479,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.38267975134342,
      86.54667450947431
     ],
     "I": [
      50.44641980227999,
      60.43490499481045
     ],
     "capability": [
      65.79924499904618,
      73.0808555865328
     ],
     "composite": {
      "A": [
       65.99814392729574,
       70.37976430138654
      ],
      "B": [
       62.310798268533894,
       68.07228955990982
      ],
      "C": [
       46.72503296526089,
       70.24484593257228
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": false,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 7.088097744249485,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.39873071167106,
     "capability": 70.78415790879176,
     "intelligence": 57.16958510591244
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 80.72765128361097,
      "cc": 69.61967049617368,
      "cc_by_type": {
       "choice": 69.61967049617368
      },
      "cc_tiered": 74.10245202356049,
      "cc_tiered_open": 82.42557001849185,
      "cc_tiered_sealed": 65.77933402862911,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.61967049617368
     },
     "noul": {
      "accuracy": 0.7173333333333334,
      "calibration": 88.35065717252257,
      "cc": 43.466666666666676,
      "cc_by_type": {
       "noul": 43.466666666666676
      },
      "cc_tiered": 44.546007779692246,
      "cc_tiered_open": 51.46376811594203,
      "cc_tiered_sealed": 37.62824744344246,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.466666666666676
     },
     "score": {
      "accuracy": 0.6933333333333334,
      "calibration": 84.11788367887968,
      "cc": 51.165347833026,
      "cc_by_type": {
       "score": 51.165347833026
      },
      "cc_tiered": 54.19533790280404,
      "cc_tiered_open": 62.74520349322689,
      "cc_tiered_sealed": 45.6454723123812,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.165347833026
     }
    },
    "raw_on_A": null,
    "run_sha256": "cadf99e94de455277604759b31028cfb14663f3b815555e5ea7909e0bb891c27",
    "set": "S+P"
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 51.86891657577944,
    "cost": 64.04636047691767,
    "intelligence": 39.60641807869822,
    "speed": 91.86196998743578
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.28942929170579373,
      "ece_score": 42.114141658841255,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 42.114141658841255,
      "score_before_time_limit": 42.114141658841255,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.07466666666666666,
      "brier": 0.17653630763852338,
      "ece": 0.16426569406488034,
      "n": 375,
      "score": 67.14686118702393,
      "score_before_time_limit": 67.14686118702393,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.592,
      "interval90_width": 0.1486666666666667,
      "n": 375,
      "nrps": 0.19154766250281158,
      "score": 46.34574688147313,
      "score_before_time_limit": 46.34574688147313,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.4407686999338628
     }
    },
    "score": 51.86891657577944
   },
   "capability": 45.737667327238825,
   "composite_ci95": {
    "A": [
     24.742350686376767,
     45.86934472760009
    ],
    "B": [
     22.27400578821329,
     42.87668159018543
    ],
    "C": [
     17.18218797665053,
     31.853711616388946
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights with no per-token tariff of their own). Exact base model Qwen/Qwen3.5-4B (author model card base_model, revision 1e7f195c) resolved under the v1.5-M2 base-model reference rule to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the frozen 25 Sep 2026 price cut-off snapshot (METHOD-v1.5-ADDENDUM-PRICING-INTERPRETATION-1 I-2/I-3); the listing was deprecated after the cut-off and stays the frozen reference. Output tokens priced at the reference output price. Replaces the larger-base proxy used at publication (Qwen3.5-9B Darkbloom endpoint, USD 0.08/0.13 per 1M). This row's usage on the common cost basis (1,479 items): 771,182 input and 1,479 output tokens. Estimated, not charged.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.04855274
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015792636916835696
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisor-4b",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 45.873028357917406,
    "I_sealed": 33.33980779947903,
    "base": 39.60641807869822,
    "excess": 5.445122814188888,
    "gap": 12.533220558438373,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 59.34553687284395,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 50.92796476879522,
       "judge": 56.49307844429796,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 60.469979296066256,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 17.803568904842024,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 31.0830694938378,
       "hard": 26.135294948151532,
       "judge": -4.274889595468578,
       "standard": 27.618054274190996
      }
     },
     "sealed|choice": {
      "cc": 53.06133527831807,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 66.66323695853484,
       "hard": 50.87225661226786,
       "judge": 49.8861047835991,
       "standard": 55.40138751238851
      }
     },
     "sealed|noul": {
      "cc": 46.958088120119044,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 66.66666666666667,
       "hard": 32.75862068965518,
       "judge": 43.28358208955223,
       "standard": 71.01449275362319
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 38.67363382263432,
       "hard": -1.500583889173468,
       "judge": -23.581442289262956,
       "standard": 15.816838400682286
      }
     }
    }
   },
   "jevbench_score": 35.33664084143631,
   "key": "decisor_4b",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "listing_note": "Cost re-estimated 12 Oct 2026 with the exact base-model reference (Qwen/Qwen3.5-4B, frozen DeepInfra 25 Sep snapshot, USD 0.03/0.15 per 1M, listing since deprecated) instead of the 9B proxy; composite A 33.52 → 35.34; Intelligence, Calibration, Speed unchanged.",
   "measured_in": "v1.6.3",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "1e7f195c2c41bbedec7d36b034001ef46476914f",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 51.86891657577944,
     "cost": 64.04636047691767,
     "intelligence": 39.60641807869822,
     "speed": 91.94556701350172
    },
    "board_release": "v1.6.43",
    "capability": 45.737667327238825,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 99,
    "capability_rank": 127,
    "composite_ci95": {
     "A": [
      24.74558117984167,
      45.875966423832466
     ],
     "B": [
      22.27609201764905,
      42.88135890458357
     ],
     "C": [
      17.184431374890043,
      31.858310016550323
     ]
    },
    "jevbench_score": 35.341565591640844,
    "measured_in": "v1.6.3",
    "method": "v1.6",
    "open_board_rank": 62,
    "rank": 92,
    "ranks": {
     "A": 92,
     "B": 91,
     "C": 93,
     "capability": 187
    },
    "scores": {
     "A": 35.341565591640844,
     "B": 32.590313637312704,
     "C": 24.542753883083925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24632775298086926,
     "p50_s_raw": 0.04816387649043463,
     "p95_s_adjusted": 0.25937609854736365,
     "p95_s_raw": 0.054688049273681816
    },
    "views": {
     "Balanced 33:33:33": 36.3832685328225,
     "Emphasis on Accuracy 60:20:20": 30.68750459394931,
     "Emphasis on Cost 20:20:60": 37.81495456321552,
     "Emphasis on Speed 20:60:20": 42.6905897612171,
     "Intelligence only": 24.851733846686635
    }
   },
   "rank": 104,
   "ranked": true,
   "ranks": {
    "A": 104,
    "B": 103,
    "C": 106,
    "capability": 208
   },
   "repo": "https://huggingface.co/underlabs/decisor-4b",
   "rescored_in": "v1.7.0",
   "score_correction": {
    "decided_on": "2026-10-12",
    "published": {
     "axes_cost": 51.35109809189328,
     "composite_ci95": {
      "A": [
       23.54664202937882,
       43.43075242179646
      ],
      "B": [
       21.491047145911537,
       41.13808580347834
      ],
      "C": [
       16.351834742624177,
       30.160244737358653
      ]
     },
     "scores": {
      "A": 33.51963655001747,
      "B": 31.333881331679464,
      "C": 23.27752538195658
     },
     "usd_per_1000": 0.04184369844489519
    },
    "reason": "Standing exact-base rule (same reference as imajev-4b, aplomb-1, Uniphore): the model card names base_model Qwen/Qwen3.5-4B, so the frozen 25 Sep DeepInfra snapshot price applies instead of the Qwen3.5-9B proxy used at publication. Official scorer path re-run on the stored raw usage; only the cost axis moves.",
    "receipt": {
     "path": "data/jevbench-score-corrections-receipts/decisor_4b-new-boot.json",
     "sha256": "382b2a72e0cdab165a8b771051c58a368ec483283007a389420df783519045e1"
    }
   },
   "scores": {
    "A": 35.33664084143631,
    "B": 32.586963245660584,
    "C": 24.539333917664106
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24737461095210164,
    "p50_s_raw": 0.048687305476050824,
    "p95_s_adjusted": 0.2632981920789461,
    "p95_s_raw": 0.056649096039473074
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.68,
       "cc": 72.43717009494861,
       "cc_by_type": {
        "choice": 28.88888888888889,
        "noul": 100.0,
        "score": 88.42262139595695
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 72.43717009494861
      },
      "calibration_gen": {
       "accuracy": 0.4375,
       "cc": -2.696359073329281,
       "cc_by_type": {
        "choice": 30.041152263374478,
        "noul": 75.0,
        "score": -113.13022948336231
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.58,
       "cc": 11.41744858021391,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 43.99999999999999,
        "score": -77.22732905610623
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 11.41744858021391
      },
      "fairness": {
       "accuracy": 0.8571428571428571,
       "cc": 84.88687477950093,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100.0,
        "score": 74.99960738935026
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 84.88687477950093
      },
      "jaggedness": {
       "accuracy": 0.46296296296296297,
       "cc": 0.611805644933618,
       "cc_by_type": {
        "choice": 45.0358999752414,
        "noul": -21.42857142857143,
        "score": -21.77191161186911
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 0.611805644933618
      },
      "llm_router": {
       "accuracy": 0.7564935064935064,
       "cc": 66.40744255292894,
       "cc_by_type": {
        "choice": 63.078569517524315,
        "noul": 77.08333333333333,
        "score": 59.06042480792917
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 66.40744255292894
      },
      "long_state": {
       "accuracy": 0.38095238095238093,
       "cc": 6.470357542728832,
       "cc_by_type": {
        "choice": 22.413793103448274,
        "noul": 0.0,
        "score": -3.0027204752617775
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 6.470357542728832
      },
      "ml_usecase": {
       "accuracy": 0.4411764705882353,
       "cc": 14.984066446494696,
       "cc_by_type": {
        "choice": 15.78947368421053,
        "noul": 50.0,
        "score": -20.837274344726442
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 14.984066446494696
      },
      "multilingual": {
       "accuracy": 0.6385964912280702,
       "cc": 42.00929251734494,
       "cc_by_type": {
        "choice": 56.102117061021175,
        "noul": 39.02439024390243,
        "score": 30.901370247111203
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 42.00929251734494
      },
      "negation_pair_gen": {
       "accuracy": 0.5714285714285714,
       "cc": 14.507954819938357,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": 41.176470588235304,
        "score": -57.652606128420224
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 14.507954819938357
      },
      "option_order": {
       "accuracy": 0.7142857142857143,
       "cc": 63.599595551061675,
       "cc_by_type": {
        "choice": 63.599595551061675
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 63.599595551061675
      },
      "perturb_pair": {
       "accuracy": 0.7741935483870968,
       "cc": 76.27284247261942,
       "cc_by_type": {
        "choice": 62.26415094339623,
        "noul": 83.33333333333333,
        "score": 83.22104314112867
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 76.27284247261942
      },
      "usecase": {
       "accuracy": 0.5637254901960784,
       "cc": 38.82789670703383,
       "cc_by_type": {
        "choice": 49.21583271097835,
        "noul": 48.0,
        "score": 19.267857410123145
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 38.82789670703383
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5294117647058824,
       "cc": 11.482547816308236,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 66.66666666666667,
        "score": -75.07616607488481
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 11.482547816308236
      },
      "cs": {
       "accuracy": 0.7777777777777778,
       "cc": 56.32183908045977,
       "cc_by_type": {
        "choice": 79.3103448275862,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 56.32183908045977
      },
      "da": {
       "accuracy": 0.5625,
       "cc": 24.36962269643198,
       "cc_by_type": {
        "choice": 53.327571305099404,
        "noul": 50.0,
        "score": -30.218703215803465
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 24.36962269643198
      },
      "de": {
       "accuracy": 0.725,
       "cc": 57.51057081567085,
       "cc_by_type": {
        "choice": 77.76862906545902,
        "noul": 63.63636363636365,
        "score": 31.12671974518988
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 57.51057081567085
      },
      "el": {
       "accuracy": 0.4117647058823529,
       "cc": 13.019548214223159,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -19.999999999999996,
        "score": 54.29673988076471
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 13.019548214223159
      },
      "en": {
       "accuracy": 0.613215859030837,
       "cc": 36.485554087248126,
       "cc_by_type": {
        "choice": 54.65150913423348,
        "noul": 53.73134328358209,
        "score": 1.073809843928808
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 36.485554087248126
      },
      "es": {
       "accuracy": 0.8,
       "cc": 69.53863836729614,
       "cc_by_type": {
        "choice": 87.95986622073579,
        "noul": 100.0,
        "score": 20.656048881152643
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 69.53863836729614
      },
      "fi": {
       "accuracy": 0.14285714285714285,
       "cc": 33.742518776617835,
       "cc_by_type": {
        "choice": -28.205128205128208,
        "noul": 100.0,
        "score": 29.43268453498171
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 33.742518776617835
      },
      "fr": {
       "accuracy": 0.75,
       "cc": 44.54072361680347,
       "cc_by_type": {
        "choice": 85.5421686746988,
        "noul": -33.333333333333336,
        "score": 81.41333550904494
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 44.54072361680347
      },
      "hi": {
       "accuracy": 0.5,
       "cc": 11.612135477060171,
       "cc_by_type": {
        "choice": 22.36598890942699,
        "noul": 66.66666666666667,
        "score": -54.196249144913146
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 11.612135477060171
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 94.57878813420577,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 100.0,
        "score": 99.75059927806217
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 94.57878813420577
      },
      "ja": {
       "accuracy": 0.5,
       "cc": -7.023794624927322,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 0.0,
        "score": -64.24423312497929
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.5555555555555556,
       "cc": 0.9298891761473259,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -100.0,
        "score": 29.456334195108635
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 0.9298891761473259
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.96890180072307,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.90670540216922
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.96890180072307
      },
      "nl": {
       "accuracy": 0.7058823529411765,
       "cc": 50.07512343166962,
       "cc_by_type": {
        "choice": 56.86900958466453,
        "noul": 50.0,
        "score": 43.35636071034432
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 50.07512343166962
      },
      "no": {
       "accuracy": 0.75,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 43.62621783952097,
       "cc_by_type": {
        "choice": 69.84924623115577,
        "noul": 50.0,
        "score": 11.029407287407144
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 43.62621783952097
      },
      "pt": {
       "accuracy": 0.7083333333333334,
       "cc": 53.41628945933892,
       "cc_by_type": {
        "choice": 36.530324400564176,
        "noul": 75.0,
        "score": 48.718543977452576
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 53.41628945933892
      },
      "sv": {
       "accuracy": 0.25,
       "cc": 11.726974968514975,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": -33.333333333333336,
        "score": 61.77850694354148
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 11.726974968514975
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 41.74690806177426,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 68.71898505488801
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 41.74690806177426
      },
      "uk": {
       "accuracy": 0.4444444444444444,
       "cc": -5.35189944151954,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -33.333333333333336,
        "score": -50.464300475096245
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 20.382580114218815,
       "cc_by_type": {
        "choice": 36.170212765957444,
        "noul": 33.33333333333333,
        "score": -8.355805756634327
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 20.382580114218815
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.652,
       "cc": 54.94735226989393,
       "cc_by_type": {
        "choice": 54.94735226989393
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 54.94735226989393
      },
      "noul": {
       "accuracy": 0.76,
       "cc": 52.0,
       "cc_by_type": {
        "noul": 52.0
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 52.0
      },
      "score": {
       "accuracy": 0.416,
       "cc": 6.985220437155759,
       "cc_by_type": {
        "score": 6.985220437155759
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 6.985220437155759
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5333333333333333,
       "cc": 13.54537966372772,
       "cc_by_type": {
        "choice": 36.39575971731449,
        "noul": 50.0,
        "score": -45.75962072613133
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 13.54537966372772
      },
      "code_linting": {
       "accuracy": 0.36363636363636365,
       "cc": 4.365900787853533,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": 19.999999999999996,
        "score": 4.8989445995854535
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 4.365900787853533
      },
      "customer_support": {
       "accuracy": 0.8461538461538461,
       "cc": 81.50290998545155,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 66.66666666666667,
        "score": 99.58119372447061
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 81.50290998545155
      },
      "demand_forecasting": {
       "accuracy": 0.3333333333333333,
       "cc": -27.435863585569564,
       "cc_by_type": {
        "choice": 35.064935064935064,
        "noul": -100.0,
        "score": -17.37252582164377
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5714285714285714,
       "cc": 31.878796844984276,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 19.999999999999996,
        "score": 8.423275780854468
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 31.878796844984276
      },
      "feature_extraction": {
       "accuracy": 0.625,
       "cc": 35.95984643622633,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 33.33333333333333,
        "score": 37.04620597534565
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 35.95984643622633
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 35.0411180236903,
       "cc_by_type": {
        "choice": 46.80851063829788,
        "noul": 60.00000000000001,
        "score": -1.6851565672269864
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 35.0411180236903
      },
      "gaming": {
       "accuracy": 0.35294117647058826,
       "cc": 4.331399313402698,
       "cc_by_type": {
        "choice": 29.742388758782198,
        "noul": 0.0,
        "score": -16.748190818574106
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 4.331399313402698
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 51.88432725992252,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 100.0,
        "score": 34.600350200820216
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 51.88432725992252
      },
      "knowledge_graphs": {
       "accuracy": 0.42857142857142855,
       "cc": 11.978713611397161,
       "cc_by_type": {
        "choice": 33.62831858407079,
        "noul": -14.28571428571429,
        "score": 16.593536535834986
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 11.978713611397161
      },
      "lead_generation": {
       "accuracy": 0.7407407407407407,
       "cc": 55.487042701031726,
       "cc_by_type": {
        "choice": 69.6969696969697,
        "noul": 100.0,
        "score": -3.235841593874511
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 55.487042701031726
      },
      "legal_compliance": {
       "accuracy": 0.4666666666666667,
       "cc": 18.106251584114915,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": -25.0,
        "score": 31.950333699713173
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 18.106251584114915
      },
      "llm_guardrails": {
       "accuracy": 0.7083333333333334,
       "cc": 51.42465114663353,
       "cc_by_type": {
        "choice": 50.81967213114754,
        "noul": 42.85714285714286,
        "score": 60.59713845161019
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 51.42465114663353
      },
      "model_routing": {
       "accuracy": 0.8029556650246306,
       "cc": 70.31448359900337,
       "cc_by_type": {
        "choice": 77.35549228630074,
        "noul": 86.44067796610169,
        "score": 47.147280544607696
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 70.31448359900337
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 31.245043349248153,
       "cc_by_type": {
        "choice": 31.81818181818182,
        "noul": 0.0,
        "score": 61.91694822956265
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 31.245043349248153
      },
      "recruiting": {
       "accuracy": 0.5882352941176471,
       "cc": 52.695732503224406,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50.0,
        "score": 45.70475236860735
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 52.695732503224406
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.497365563076894,
       "cc_by_type": {
        "choice": 41.32029339853301,
        "noul": 33.33333333333333,
        "score": -7.1615300426356665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 22.497365563076894
      },
      "scientific_discovery": {
       "accuracy": 0.5238095238095238,
       "cc": 23.782218083000426,
       "cc_by_type": {
        "choice": 41.747572815533985,
        "noul": 0.0,
        "score": 29.59908143346729
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 23.782218083000426
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 74.1814325078106,
       "cc_by_type": {
        "choice": 88.39458413926498,
        "noul": 100.0,
        "score": 34.1497133841668
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 74.1814325078106
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      46.9497762071451,
      56.41045180978028
     ],
     "I": [
      34.24026015889371,
      44.09619091401736
     ],
     "capability": [
      41.14796787605721,
      49.56906133902689
     ],
     "composite": {
      "A": [
       24.742350686376767,
       45.86934472760009
      ],
      "B": [
       22.27400578821329,
       42.87668159018543
      ],
      "C": [
       17.18218797665053,
       31.853711616388946
      ]
     },
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 51.86891657577944,
     "capability": 45.737667327238825,
     "intelligence": 39.60641807869822
    },
    "per_type": {
     "choice": {
      "accuracy": 0.652,
      "calibration": 42.114141658841255,
      "cc": 54.94735226989393,
      "cc_by_type": {
       "choice": 54.94735226989393
      },
      "cc_tiered": 56.20343607558101,
      "cc_tiered_open": 59.34553687284395,
      "cc_tiered_sealed": 53.06133527831807,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.94735226989393
     },
     "noul": {
      "accuracy": 0.76,
      "calibration": 67.14686118702393,
      "cc": 52.0,
      "cc_by_type": {
       "noul": 52.0
      },
      "cc_tiered": 53.71403370809265,
      "cc_tiered_open": 60.469979296066256,
      "cc_tiered_sealed": 46.958088120119044,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 52.0
     },
     "score": {
      "accuracy": 0.416,
      "calibration": 46.34574688147313,
      "cc": 6.985220437155759,
      "cc_by_type": {
       "score": 6.985220437155759
      },
      "cc_tiered": 8.901784452421012,
      "cc_tiered_open": 17.803568904842024,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 6.985220437155759
     }
    },
    "raw_on_A": {
     "calibration": 54.13393694270948,
     "capability": 49.11000935691333,
     "intelligence": 44.08608177111718
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 36.376309766578665,
    "Emphasis on Accuracy 60:20:20": 30.68453399342297,
    "Emphasis on Cost 20:20:60": 37.81044391968969,
    "Emphasis on Speed 20:60:20": 42.673348331458556,
    "Intelligence only": 24.851733846686635
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 91.10757097649757,
    "cost": 49.20157807658991,
    "intelligence": 71.89886397161271,
    "speed": 88.47496650745126
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05187038243550904,
      "ece_score": 89.6259235128982,
      "mean_tvd": null,
      "n": 735,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 89.6259235128982,
      "score_before_time_limit": 89.6259235128982,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.07263564703720832,
      "ece": 0.03721686055451093,
      "n": 373,
      "score": 92.55662788909783,
      "score_before_time_limit": 92.55662788909783,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9703504043126685,
      "interval90_width": 0.29222821203953275,
      "n": 371,
      "nrps": 0.06110092445996998,
      "score": 91.1401615274967,
      "score_before_time_limit": 91.1401615274967,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0580479224950481
     }
    },
    "score": 91.10757097649757
   },
   "capability": 81.50321747405513,
   "composite_ci95": {
    "A": [
     67.0232591067018,
     69.3194421357184
    ],
    "B": [
     66.65031985722759,
     69.98840326920889
    ],
    "C": [
     67.0232591067018,
     69.3194421357184
    ]
   },
   "cost": {
    "basis": "Public exact base-model OpenRouter reference for google/gemma-4-31b-it, 2026-10-09: cheapest paid DeepInfra Turbo $0.09/M input, $0.34/M output. Jeff uses native bf16 single forward-pass decision readout with zero generated output tokens. Reference tariff is not an actual Jeff hosting bill. Primary source https://openrouter.ai/api/v1/models/google/gemma-4-31b-it/endpoints; source receipt retained.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.048658379999999994
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04934926977687626
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jeff-1.0-Large",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.94592822865295,
    "I_sealed": 69.85179971457248,
    "base": 71.89886397161271,
    "excess": -2.993969230169018,
    "gap": 4.094128514080467,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 76.92440600606106,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 68.35860250494397,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 85.29951690821257,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 70.0,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 59.61386177168524,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.31995405652513,
       "hard": 69.76831247393966,
       "judge": 63.52914643184884,
       "standard": 27.578987234511022
      }
     },
     "sealed|choice": {
      "cc": 81.31363869979835,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 88.88774565284494,
       "hard": 78.22172200337647,
       "judge": 79.04328018223235,
       "standard": 87.11595639246778
      }
     },
     "sealed|noul": {
      "cc": 77.2270301416456,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 87.5,
       "hard": 70.6896551724138,
       "judge": 73.13432835820895,
       "standard": 91.30434782608697
      }
     },
     "sealed|score": {
      "cc": 51.01473030227352,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 79.67608755640215,
       "hard": 44.71435253600092,
       "judge": 42.59576964785805,
       "standard": 61.913248189377576
      }
     }
    }
   },
   "jevbench_score": 68.53936803256003,
   "key": "jeff_1_0_large",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "275325a5eab52cce4f109a09ee3832f2eb89fe2f",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.10757097649757,
     "cost": 49.20157807658991,
     "intelligence": 71.89886397161271,
     "speed": 88.82411852274376
    },
    "board_release": "v1.6.43",
    "capability": 81.50321747405513,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 5,
    "capability_rank": 9,
    "composite_ci95": {
     "A": [
      67.07482591953416,
      69.37460423313809
     ],
     "B": [
      66.69110907316322,
      70.03338191380395
     ],
     "C": [
      67.07482591953416,
      69.37460423313809
     ]
    },
    "jevbench_score": 68.59329511867139,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": 15,
    "rank": 32,
    "ranks": {
     "A": 32,
     "B": 25,
     "C": 28,
     "capability": 20
    },
    "scores": {
     "A": 68.59329511867139,
     "B": 68.79639624966302,
     "C": 68.59329511867139
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3101211428642273,
     "p50_s_raw": 0.08006057143211365,
     "p95_s_adjusted": 0.4227238700725138,
     "p95_s_raw": 0.1363619350362569
    },
    "views": {
     "Balanced 33:33:33": 63.85752832522542,
     "Emphasis on Accuracy 60:20:20": 66.04447649061674,
     "Emphasis on Cost 20:20:60": 56.205881793219106,
     "Emphasis on Speed 20:60:20": 71.19191780408633,
     "Intelligence only": 69.62097236625647
    }
   },
   "rank": 38,
   "ranked": true,
   "ranks": {
    "A": 38,
    "B": 27,
    "C": 32,
    "capability": 23
   },
   "repo": "https://huggingface.co/jgeuter/Jeff-1.0-Large",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.53936803256003,
    "B": 68.75299197819241,
    "C": 68.53936803256003
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.31703930497169497,
    "p50_s_raw": 0.08351965248584747,
    "p95_s_adjusted": 0.4481157846748829,
    "p95_s_raw": 0.14905789233744146
   },
   "status": {
    "answered_ok": 1479,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.96,
       "cc": 96.9377101225352,
       "cc_by_type": {
        "choice": 91.1111111111111,
        "noul": 100.0,
        "score": 99.70201925649454
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 96.9377101225352
      },
      "calibration_gen": {
       "accuracy": 0.78125,
       "cc": 57.561533071029515,
       "cc_by_type": {
        "choice": 75.30864197530865,
        "noul": 66.66666666666667,
        "score": 30.709290571113236
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 57.561533071029515
      },
      "composite_gen": {
       "accuracy": 0.92,
       "cc": 79.81401737255631,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 60.00000000000001,
        "score": 82.69408463799414
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 79.81401737255631
      },
      "fairness": {
       "accuracy": 0.9285714285714286,
       "cc": 91.50025446299001,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100.0,
        "score": 94.8397464398175
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 91.50025446299001
      },
      "jaggedness": {
       "accuracy": 0.6604938271604939,
       "cc": 47.147992393452604,
       "cc_by_type": {
        "choice": 64.34761079475119,
        "noul": 50.0,
        "score": 27.09636638560662
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 47.147992393452604
      },
      "llm_router": {
       "accuracy": 0.9318181818181818,
       "cc": 85.66561133970038,
       "cc_by_type": {
        "choice": 91.97360206902702,
        "noul": 97.91666666666667,
        "score": 67.10656528340743
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 85.66561133970038
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -131.99233716475095,
       "cc_by_type": {
        "choice": -29.310344827586214,
        "noul": -100.0,
        "score": -266.66666666666663
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.8529411764705882,
       "cc": 77.55088834993914,
       "cc_by_type": {
        "choice": 89.47368421052633,
        "noul": 75.0,
        "score": 68.1789808392911
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 77.55088834993914
      },
      "multilingual": {
       "accuracy": 0.8350877192982457,
       "cc": 75.39071926905552,
       "cc_by_type": {
        "choice": 79.45205479452055,
        "noul": 78.04878048780488,
        "score": 68.67132252484116
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 75.39071926905552
      },
      "negation_pair_gen": {
       "accuracy": 0.9183673469387755,
       "cc": 82.11185908219272,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 94.11764705882352,
        "score": 58.88459685442133
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 82.11185908219272
      },
      "option_order": {
       "accuracy": 0.9365079365079365,
       "cc": 91.91102123356926,
       "cc_by_type": {
        "choice": 91.91102123356926
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 91.91102123356926
      },
      "perturb_pair": {
       "accuracy": 0.9032258064516129,
       "cc": 82.33336907809512,
       "cc_by_type": {
        "choice": 87.42138364779876,
        "noul": 83.33333333333333,
        "score": 76.24539025315326
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 82.33336907809512
      },
      "usecase": {
       "accuracy": 0.7549019607843137,
       "cc": 66.3546834985278,
       "cc_by_type": {
        "choice": 76.10156833457805,
        "noul": 76.0,
        "score": 46.96248216100536
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 66.3546834985278
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8235294117647058,
       "cc": 62.110383554231476,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 5.378769710313469
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 62.110383554231476
      },
      "cs": {
       "accuracy": 0.6666666666666666,
       "cc": 45.97701149425287,
       "cc_by_type": {
        "choice": 58.62068965517241,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 45.97701149425287
      },
      "da": {
       "accuracy": 0.75,
       "cc": 55.709291217777256,
       "cc_by_type": {
        "choice": 68.88504753673294,
        "noul": 50.0,
        "score": 48.24282611659883
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 55.709291217777256
      },
      "de": {
       "accuracy": 0.95,
       "cc": 91.29312298128339,
       "cc_by_type": {
        "choice": 92.58954302181968,
        "noul": 100.0,
        "score": 81.28982592203047
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 91.29312298128339
      },
      "el": {
       "accuracy": 0.7647058823529411,
       "cc": 58.66399707665688,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 19.999999999999996,
        "score": 94.08722932520874
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 58.66399707665688
      },
      "en": {
       "accuracy": 0.8290748898678414,
       "cc": 71.06721855084093,
       "cc_by_type": {
        "choice": 80.78832406671961,
        "noul": 79.8507462686567,
        "score": 52.562585317146485
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 71.06721855084093
      },
      "es": {
       "accuracy": 0.9333333333333333,
       "cc": 88.55170639268187,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0,
        "score": 65.65511917804558
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 88.55170639268187
      },
      "fi": {
       "accuracy": 1.0,
       "cc": 93.91862104229479,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 81.75586312688439
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 93.91862104229479
      },
      "fr": {
       "accuracy": 1.0,
       "cc": 98.41120186775261,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.23360560325787
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 98.41120186775261
      },
      "hi": {
       "accuracy": 0.8846153846153846,
       "cc": 84.11210683870443,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 52.3363205161133
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 84.11210683870443
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.8461538461538461,
       "cc": 79.18660535597223,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 100.0,
        "score": 53.57405094336154
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 79.18660535597223
      },
      "ja": {
       "accuracy": 0.6818181818181818,
       "cc": 48.67304788730957,
       "cc_by_type": {
        "choice": 71.58642462509866,
        "noul": 75.0,
        "score": -0.567280963169936
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 48.67304788730957
      },
      "ko": {
       "accuracy": 0.8888888888888888,
       "cc": 90.81791628019677,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100.0,
        "score": 99.12041550725698
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 90.81791628019677
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.4882036536204,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.4646109608612
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.4882036536204
      },
      "nl": {
       "accuracy": 0.8823529411764706,
       "cc": 76.76055951504456,
       "cc_by_type": {
        "choice": 85.62300319488817,
        "noul": 100.0,
        "score": 44.65867535024552
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 76.76055951504456
      },
      "no": {
       "accuracy": 0.75,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 64.15309188662947,
       "cc_by_type": {
        "choice": 92.46231155778895,
        "noul": 50.0,
        "score": 49.99696410209948
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 64.15309188662947
      },
      "pt": {
       "accuracy": 0.7916666666666666,
       "cc": 67.0299432480883,
       "cc_by_type": {
        "choice": 61.918194640338506,
        "noul": 75.0,
        "score": 64.17163510392639
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 67.0299432480883
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 34.30530473285898,
       "cc_by_type": {
        "choice": 37.8238341968912,
        "noul": 33.33333333333333,
        "score": 31.758746668352412
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 34.30530473285898
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 76.84926416089685,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 52.28692291747318
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 76.84926416089685
      },
      "uk": {
       "accuracy": 0.6666666666666666,
       "cc": 70.01912003849547,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 74.57348914774444
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 70.01912003849547
      },
      "zh": {
       "accuracy": 0.9166666666666666,
       "cc": 77.21435846743238,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 31.64307540229713
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 77.21435846743238
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8546666666666667,
       "cc": 81.18490956865303,
       "cc_by_type": {
        "choice": 81.18490956865303
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.18490956865303
      },
      "noul": {
       "accuracy": 0.9013333333333333,
       "cc": 80.26666666666667,
       "cc_by_type": {
        "noul": 80.26666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 80.26666666666667
      },
      "score": {
       "accuracy": 0.7226666666666667,
       "cc": 55.788639506853855,
       "cc_by_type": {
        "score": 55.788639506853855
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 55.788639506853855
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6666666666666666,
       "cc": 49.563854302825206,
       "cc_by_type": {
        "choice": 57.597173144876315,
        "noul": 50.0,
        "score": 41.0943897635993
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 49.563854302825206
      },
      "code_linting": {
       "accuracy": 0.8181818181818182,
       "cc": 84.20240094634025,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 52.60720283902074
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 84.20240094634025
      },
      "customer_support": {
       "accuracy": 0.7692307692307693,
       "cc": 43.650863010018696,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": -25.5691501003787
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 43.650863010018696
      },
      "demand_forecasting": {
       "accuracy": 0.38095238095238093,
       "cc": 19.258228775832507,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": 50.0,
        "score": -14.303235750424559
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 19.258228775832507
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8571428571428571,
       "cc": 75.10276024397294,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 65.30828073191881
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 75.10276024397294
      },
      "feature_extraction": {
       "accuracy": 0.8125,
       "cc": 71.104810968882,
       "cc_by_type": {
        "choice": 68.75,
        "noul": 100.0,
        "score": 44.56443290664603
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 71.104810968882
      },
      "financial_crime": {
       "accuracy": 0.7142857142857143,
       "cc": 74.39743035918184,
       "cc_by_type": {
        "choice": 89.36170212765957,
        "noul": 100.0,
        "score": 33.83058894988594
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.39743035918184
      },
      "gaming": {
       "accuracy": 0.7058823529411765,
       "cc": 43.987755987145654,
       "cc_by_type": {
        "choice": 57.84543325526932,
        "noul": 0.0,
        "score": 74.11783470616764
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 43.987755987145654
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 68.17072508006011,
       "cc_by_type": {
        "choice": 68.42105263157895,
        "noul": 100.0,
        "score": 36.09112260860137
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 68.17072508006011
      },
      "knowledge_graphs": {
       "accuracy": 0.8095238095238095,
       "cc": 71.46545854521419,
       "cc_by_type": {
        "choice": 73.45132743362832,
        "noul": 71.42857142857142,
        "score": 69.51647677344282
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 71.46545854521419
      },
      "lead_generation": {
       "accuracy": 0.9259259259259259,
       "cc": 78.65664565502806,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 35.96993696508418
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 78.65664565502806
      },
      "legal_compliance": {
       "accuracy": 0.8666666666666667,
       "cc": 76.81444528324634,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 75.0,
        "score": 81.75912532342325
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 76.81444528324634
      },
      "llm_guardrails": {
       "accuracy": 0.7916666666666666,
       "cc": 72.51849215446394,
       "cc_by_type": {
        "choice": 83.60655737704917,
        "noul": 71.42857142857142,
        "score": 62.52034765777125
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 72.51849215446394
      },
      "model_routing": {
       "accuracy": 0.9261083743842364,
       "cc": 83.17577360184441,
       "cc_by_type": {
        "choice": 95.23273521816857,
        "noul": 100.0,
        "score": 54.294585587364665
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 83.17577360184441
      },
      "moderation_trust_safety": {
       "accuracy": 0.9166666666666666,
       "cc": 81.14209861079502,
       "cc_by_type": {
        "choice": 90.25974025974027,
        "noul": 66.66666666666667,
        "score": 86.49988890597811
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 81.14209861079502
      },
      "recruiting": {
       "accuracy": 0.7647058823529411,
       "cc": 66.11227351729069,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50.0,
        "score": 85.9543754108062
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 66.11227351729069
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 63.987715661714255,
       "cc_by_type": {
        "choice": 55.99022004889975,
        "noul": 100.0,
        "score": 35.97292693624302
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 63.987715661714255
      },
      "scientific_discovery": {
       "accuracy": 0.7619047619047619,
       "cc": 59.91676158127726,
       "cc_by_type": {
        "choice": 70.873786407767,
        "noul": 50.0,
        "score": 58.87649833606476
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 59.91676158127726
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 82.16687648931749,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 46.50062946795245
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 82.16687648931749
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.33110758644384,
      92.2500086656397
     ],
     "I": [
      67.09450470102875,
      75.25396599769745
     ],
     "capability": [
      77.88525338666481,
      83.2725659810476
     ],
     "composite": {
      "A": [
       67.0232591067018,
       69.3194421357184
      ],
      "B": [
       66.65031985722759,
       69.98840326920889
      ],
      "C": [
       67.0232591067018,
       69.3194421357184
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.10757097649757,
     "capability": 81.50321747405513,
     "intelligence": 71.89886397161271
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 89.6259235128982,
      "cc": 81.18490956865303,
      "cc_by_type": {
       "choice": 81.18490956865303
      },
      "cc_tiered": 79.1190223529297,
      "cc_tiered_open": 76.92440600606106,
      "cc_tiered_sealed": 81.31363869979835,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.18490956865303
     },
     "noul": {
      "accuracy": 0.9013333333333333,
      "calibration": 92.55662788909783,
      "cc": 80.26666666666667,
      "cc_by_type": {
       "noul": 80.26666666666667
      },
      "cc_tiered": 81.26327352492908,
      "cc_tiered_open": 85.29951690821257,
      "cc_tiered_sealed": 77.2270301416456,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 80.26666666666667
     },
     "score": {
      "accuracy": 0.7226666666666667,
      "calibration": 91.1401615274967,
      "cc": 55.788639506853855,
      "cc_by_type": {
       "score": 55.788639506853855
      },
      "cc_tiered": 55.31429603697938,
      "cc_tiered_open": 59.61386177168524,
      "cc_tiered_sealed": 51.01473030227352,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.788639506853855
     }
    },
    "raw_on_A": {
     "calibration": 90.61114153337692,
     "capability": 81.96187686080913,
     "intelligence": 73.31261218824132
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.020000000000000018,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.79522313956008,
    "Emphasis on Accuracy 60:20:20": 66.00447418190907,
    "Emphasis on Cost 20:20:60": 56.17690737182881,
    "Emphasis on Speed 20:60:20": 71.05266386423217,
    "Intelligence only": 69.62097236625647
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 80.89039994344657,
    "cost": 51.82161587751452,
    "intelligence": 51.08240263971605,
    "speed": 78.38456523323399
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06046272357834501,
      "ece_score": 87.907455284331,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 87.907455284331,
      "score_before_time_limit": 87.907455284331,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.16334625066666603,
      "ece": 0.11029866666666785,
      "n": 375,
      "score": 77.94026666666642,
      "score_before_time_limit": 77.94026666666642,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8826666666666667,
      "interval90_width": 0.5564444444444444,
      "n": 375,
      "nrps": 0.1216478762698887,
      "score": 76.82347787934233,
      "score_before_time_limit": 76.82347787934233,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1709412830716323
     }
    },
    "score": 80.89039994344657
   },
   "capability": 65.9864012915813,
   "composite_ci95": {
    "A": [
     47.13172518767568,
     63.968615083981284
    ],
    "B": [
     44.15812168678088,
     61.96385581536533
    ],
    "C": [
     32.73036471366367,
     54.09215363001676
    ]
   },
   "cost": {
    "basis": "Self-hosted open weights, no bookable tariff. v1.5-M2 base-model reference: the model card names base_model Qwen/Qwen3.5-4B. Per the frozen v1.5 precedent (METHOD-v1.5-ADDENDUM-PRICING-INTERPRETATION-1.md I-2/I-3), Qwen3.5-4B rows use DeepInfra Qwen/Qwen3.5-4B at USD 0.03/M input and USD 0.15/M output (frozen 25 Sep 2026 price cut-off snapshot; listing later deprecated, frozen reference retained per I-3). Read from the official method files on 2026-10-08; no fresh provider page was fetched. Output tokens priced at the reference output rate. Estimate, not a bookable tariff.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.04652426
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04035953346855983
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Metask rain 4B",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 56.1234361269253,
    "I_sealed": 46.041369152506796,
    "base": 51.08240263971605,
    "excess": 2.993969230169018,
    "gap": 10.082066974418503,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 73.60321904242878,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 66.02705253224286,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 54.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 39.825060352839884,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.599209143896296,
       "hard": 26.675379512412224,
       "judge": 65.98556402232718,
       "standard": 25.996592133936048
      }
     },
     "sealed|choice": {
      "cc": 58.676637540616824,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 68.51527934972734,
       "hard": 52.89814293753516,
       "judge": 62.64236902050115,
       "standard": 59.36570862239843
      }
     },
     "sealed|noul": {
      "cc": 56.6951598827452,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 66.66666666666667,
       "hard": 48.27586206896552,
       "judge": 49.25373134328359,
       "standard": 79.71014492753623
      }
     },
     "sealed|score": {
      "cc": 22.752310034158377,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 28.051523454106352,
       "hard": 29.62563522191024,
       "judge": 13.23325264200742,
       "standard": 20.63463903690709
      }
     }
    }
   },
   "jevbench_score": 62.506641362844796,
   "key": "metask-jev-rain-4b",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "ea20fe85b28733b1522721dec119bec50947a869",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 80.89039994344657,
     "cost": 51.82161587751452,
     "intelligence": 51.08240263971605,
     "speed": 79.59790373470649
    },
    "board_release": "v1.6.43",
    "capability": 65.9864012915813,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 35,
    "capability_rank": 56,
    "composite_ci95": {
     "A": [
      47.26843551984421,
      64.16817644128109
     ],
     "B": [
      44.254055214557084,
      62.11354987809795
     ],
     "C": [
      32.82530244433626,
      54.26142073872429
     ]
    },
    "jevbench_score": 62.69717160365172,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": 31,
    "rank": 52,
    "ranks": {
     "A": 52,
     "B": 52,
     "C": 57,
     "capability": 98
    },
    "scores": {
     "A": 62.69717160365172,
     "B": 59.97005503140675,
     "C": 45.44520642674837
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26540608701761814,
     "p50_s_raw": 0.05770304350880906,
     "p95_s_adjusted": 4.133317315817111,
     "p95_s_raw": 1.9916586579085556
    },
    "views": {
     "Balanced 33:33:33": 58.3245403206291,
     "Emphasis on Accuracy 60:20:20": 55.194490971444196,
     "Emphasis on Cost 20:20:60": 55.53688383211001,
     "Emphasis on Speed 20:60:20": 65.30602465884293,
     "Intelligence only": 51.08240263971605
    }
   },
   "rank": 58,
   "ranked": true,
   "ranks": {
    "A": 58,
    "B": 58,
    "C": 65,
    "capability": 114
   },
   "repo": "https://huggingface.co/wayfind/metask-jev-4b-policy-mix",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 62.506641362844796,
    "B": 59.83050276635585,
    "C": 45.30710312954151
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3165951920440421,
    "p50_s_raw": 0.08329759602202103,
    "p95_s_adjusted": 4.581832004513126,
    "p95_s_raw": 2.215916002256563
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.96,
       "cc": 94.84452702447224,
       "cc_by_type": {
        "choice": 91.1111111111111,
        "noul": 100.0,
        "score": 93.42246996230561
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 94.84452702447224
      },
      "calibration_gen": {
       "accuracy": 0.5520833333333334,
       "cc": 26.715869992998762,
       "cc_by_type": {
        "choice": 46.50205761316872,
        "noul": 58.33333333333333,
        "score": -24.687780967505766
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 26.715869992998762
      },
      "composite_gen": {
       "accuracy": 0.59,
       "cc": 30.609363116297118,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 60.00000000000001,
        "score": -19.39142284623059
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 30.609363116297118
      },
      "fairness": {
       "accuracy": 0.9761904761904762,
       "cc": 96.75272495371574,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 90.25817486114721
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 96.75272495371574
      },
      "jaggedness": {
       "accuracy": 0.5432098765432098,
       "cc": 24.77815252618288,
       "cc_by_type": {
        "choice": 42.06486754147067,
        "noul": 42.85714285714286,
        "score": -10.587552820064893
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 24.77815252618288
      },
      "llm_router": {
       "accuracy": 0.8084415584415584,
       "cc": 69.36316339410013,
       "cc_by_type": {
        "choice": 75.92080620708107,
        "noul": 85.41666666666667,
        "score": 46.75201730855263
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 69.36316339410013
      },
      "long_state": {
       "accuracy": 0.38095238095238093,
       "cc": -7.847225627137181,
       "cc_by_type": {
        "choice": 22.413793103448274,
        "noul": 0.0,
        "score": -45.95546998485982
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5882352941176471,
       "cc": 41.08191998802071,
       "cc_by_type": {
        "choice": 47.36842105263158,
        "noul": 50.0,
        "score": 25.877338911430535
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 41.08191998802071
      },
      "multilingual": {
       "accuracy": 0.6701754385964912,
       "cc": 51.06107828522669,
       "cc_by_type": {
        "choice": 57.97011207970113,
        "noul": 46.34146341463414,
        "score": 48.8716593613448
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 51.06107828522669
      },
      "negation_pair_gen": {
       "accuracy": 0.6428571428571429,
       "cc": 36.463286789996,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 41.176470588235304,
        "score": 1.546723115086024
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 36.463286789996
      },
      "option_order": {
       "accuracy": 0.7619047619047619,
       "cc": 69.66632962588473,
       "cc_by_type": {
        "choice": 69.66632962588473
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 69.66632962588473
      },
      "perturb_pair": {
       "accuracy": 0.7741935483870968,
       "cc": 65.19271120746511,
       "cc_by_type": {
        "choice": 71.69811320754718,
        "noul": 66.66666666666667,
        "score": 57.21335374818144
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 65.19271120746511
      },
      "usecase": {
       "accuracy": 0.5686274509803921,
       "cc": 40.31421175143449,
       "cc_by_type": {
        "choice": 53.696788648244976,
        "noul": 39.99999999999999,
        "score": 27.2458466060585
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 40.31421175143449
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4117647058823529,
       "cc": -0.07450502189792256,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -43.08065792283662
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5555555555555556,
       "cc": 35.632183908045974,
       "cc_by_type": {
        "choice": 37.93103448275862,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 35.632183908045974
      },
      "da": {
       "accuracy": 0.6875,
       "cc": 60.90270576587461,
       "cc_by_type": {
        "choice": 53.327571305099404,
        "noul": 100.0,
        "score": 29.380545992524432
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 60.90270576587461
      },
      "de": {
       "accuracy": 0.775,
       "cc": 62.41736197307345,
       "cc_by_type": {
        "choice": 70.3581720872787,
        "noul": 81.81818181818181,
        "score": 35.07573201375983
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 62.41736197307345
      },
      "el": {
       "accuracy": 0.7647058823529411,
       "cc": 73.37747350577037,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 60.00000000000001,
        "score": 79.18003956493014
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 73.37747350577037
      },
      "en": {
       "accuracy": 0.6713656387665198,
       "cc": 48.42563024796656,
       "cc_by_type": {
        "choice": 62.24682287529783,
        "noul": 61.19402985074627,
        "score": 21.836038017855586
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 48.42563024796656
      },
      "es": {
       "accuracy": 0.7666666666666667,
       "cc": 65.47988597079195,
       "cc_by_type": {
        "choice": 63.879598662207364,
        "noul": 81.81818181818181,
        "score": 50.74187743198667
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 65.47988597079195
      },
      "fi": {
       "accuracy": 0.0,
       "cc": -24.70996081033493,
       "cc_by_type": {
        "choice": -28.205128205128208,
        "noul": -100.0,
        "score": 54.075245774123424
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.625,
       "cc": 29.6779881965786,
       "cc_by_type": {
        "choice": 56.62650602409638,
        "noul": -33.333333333333336,
        "score": 65.74079189897277
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 29.6779881965786
      },
      "hi": {
       "accuracy": 0.6153846153846154,
       "cc": 40.80657497403471,
       "cc_by_type": {
        "choice": 44.547134935304996,
        "noul": 66.66666666666667,
        "score": 11.205923320132472
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 40.80657497403471
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 92.98419698867792,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 100.0,
        "score": 94.96682584147862
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 92.98419698867792
      },
      "ja": {
       "accuracy": 0.45454545454545453,
       "cc": 7.879051793920785,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 0.0,
        "score": -19.53569386843497
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 7.879051793920785
      },
      "ko": {
       "accuracy": 0.6666666666666666,
       "cc": 64.53133627110843,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100.0,
        "score": 20.260675479991942
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 64.53133627110843
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.8719181249276,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.61575437478281
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.8719181249276
      },
      "nl": {
       "accuracy": 0.4117647058823529,
       "cc": -15.949562639532397,
       "cc_by_type": {
        "choice": 28.115015974440897,
        "noul": -50.0,
        "score": -25.96370389303808
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.75,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 60.219275506836446,
       "cc_by_type": {
        "choice": 84.92462311557787,
        "noul": 66.66666666666667,
        "score": 29.0665367382648
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 60.219275506836446
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 74.11075635925,
       "cc_by_type": {
        "choice": 87.30606488011284,
        "noul": 75.0,
        "score": 60.026204197637156
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 74.11075635925
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 29.78604643849067,
       "cc_by_type": {
        "choice": 68.9119170984456,
        "noul": -33.333333333333336,
        "score": 53.77955555035974
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 29.78604643849067
      },
      "tr": {
       "accuracy": 0.5454545454545454,
       "cc": 5.547657907361341,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": -39.87876540835076
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 5.547657907361341
      },
      "uk": {
       "accuracy": 0.8888888888888888,
       "cc": 80.82177970414698,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 74.72340362856995
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 80.82177970414698
      },
      "zh": {
       "accuracy": 0.6666666666666666,
       "cc": 52.23199799021912,
       "cc_by_type": {
        "choice": 36.170212765957444,
        "noul": 66.66666666666667,
        "score": 53.859114538033225
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 52.23199799021912
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.704,
       "cc": 61.67935710312817,
       "cc_by_type": {
        "choice": 61.67935710312817
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.67935710312817
      },
      "noul": {
       "accuracy": 0.7946666666666666,
       "cc": 58.93333333333332,
       "cc_by_type": {
        "noul": 58.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 58.93333333333332
      },
      "score": {
       "accuracy": 0.48533333333333334,
       "cc": 26.7077945049382,
       "cc_by_type": {
        "score": 26.7077945049382
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 26.7077945049382
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5666666666666667,
       "cc": 28.124915716071694,
       "cc_by_type": {
        "choice": 36.39575971731449,
        "noul": 50.0,
        "score": -2.021012569099412
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 28.124915716071694
      },
      "code_linting": {
       "accuracy": 0.4090909090909091,
       "cc": 28.254375685619937,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 60.00000000000001,
        "score": -0.70271145245695
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 28.254375685619937
      },
      "customer_support": {
       "accuracy": 0.6923076923076923,
       "cc": 59.49337631484335,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 88.62505648076197
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 59.49337631484335
      },
      "demand_forecasting": {
       "accuracy": 0.5238095238095238,
       "cc": 21.86363398640653,
       "cc_by_type": {
        "choice": 61.03896103896104,
        "noul": 0.0,
        "score": 4.5519409202585575
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 21.86363398640653
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7142857142857143,
       "cc": 53.51614932057469,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 60.00000000000001,
        "score": 33.335333207625695
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 53.51614932057469
      },
      "feature_extraction": {
       "accuracy": 0.5625,
       "cc": 5.923903534888588,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -33.333333333333336,
        "score": 13.605043937999096
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 5.923903534888588
      },
      "financial_crime": {
       "accuracy": 0.6071428571428571,
       "cc": 48.94759929822106,
       "cc_by_type": {
        "choice": 78.72340425531915,
        "noul": 60.00000000000001,
        "score": 8.119393639343997
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 48.94759929822106
      },
      "gaming": {
       "accuracy": 0.35294117647058826,
       "cc": 24.14019231524544,
       "cc_by_type": {
        "choice": 43.79391100702576,
        "noul": 0.0,
        "score": 28.62666593871055
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 24.14019231524544
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 25.272604681173657,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 0.0,
        "score": 54.765182464573606
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 25.272604681173657
      },
      "knowledge_graphs": {
       "accuracy": 0.42857142857142855,
       "cc": 18.187989919020836,
       "cc_by_type": {
        "choice": 20.353982300884958,
        "noul": 14.28571428571428,
        "score": 19.92427317046327
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 18.187989919020836
      },
      "lead_generation": {
       "accuracy": 0.6296296296296297,
       "cc": 55.683058307434294,
       "cc_by_type": {
        "choice": 49.4949494949495,
        "noul": 80.0,
        "score": 37.55422542735337
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 55.683058307434294
      },
      "legal_compliance": {
       "accuracy": 0.7333333333333333,
       "cc": 49.52547609201625,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 50.0,
        "score": 24.892217749732946
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 49.52547609201625
      },
      "llm_guardrails": {
       "accuracy": 0.75,
       "cc": 46.187284107298744,
       "cc_by_type": {
        "choice": 50.81967213114754,
        "noul": 14.28571428571428,
        "score": 73.4564659050344
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 46.187284107298744
      },
      "model_routing": {
       "accuracy": 0.7783251231527094,
       "cc": 63.54604802056192,
       "cc_by_type": {
        "choice": 77.35549228630074,
        "noul": 83.05084745762711,
        "score": 30.231804317757916
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 63.54604802056192
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 19.915539893895183,
       "cc_by_type": {
        "choice": 31.81818181818182,
        "noul": 0.0,
        "score": 27.92843786350373
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 19.915539893895183
      },
      "recruiting": {
       "accuracy": 0.5882352941176471,
       "cc": 67.38856406151355,
       "cc_by_type": {
        "choice": 43.573667711598745,
        "noul": 100.0,
        "score": 58.592024472941894
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 67.38856406151355
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 25.997491018065205,
       "cc_by_type": {
        "choice": 70.6601466992665,
        "noul": -33.333333333333336,
        "score": 40.665659688262444
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 25.997491018065205
      },
      "scientific_discovery": {
       "accuracy": 0.38095238095238093,
       "cc": 17.543893040423583,
       "cc_by_type": {
        "choice": 12.621359223300967,
        "noul": 0.0,
        "score": 40.01031989796978
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 17.543893040423583
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 70.40470952339105,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 11.214128570173177
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 70.40470952339105
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.5497718269952,
      83.39564578949536
     ],
     "I": [
      44.504765007776726,
      55.16844250289851
     ],
     "capability": [
      61.09493578058464,
      68.36932275256177
     ],
     "composite": {
      "A": [
       47.13172518767568,
       63.968615083981284
      ],
      "B": [
       44.15812168678088,
       61.96385581536533
      ],
      "C": [
       32.73036471366367,
       54.09215363001676
      ]
     },
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.89039994344657,
     "capability": 65.9864012915813,
     "intelligence": 51.08240263971605
    },
    "per_type": {
     "choice": {
      "accuracy": 0.704,
      "calibration": 87.907455284331,
      "cc": 61.67935710312817,
      "cc_by_type": {
       "choice": 61.67935710312817
      },
      "cc_tiered": 66.1399282915228,
      "cc_tiered_open": 73.60321904242878,
      "cc_tiered_sealed": 58.676637540616824,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.67935710312817
     },
     "noul": {
      "accuracy": 0.7946666666666666,
      "calibration": 77.94026666666642,
      "cc": 58.93333333333332,
      "cc_by_type": {
       "noul": 58.93333333333332
      },
      "cc_tiered": 55.818594434126226,
      "cc_tiered_open": 54.94202898550725,
      "cc_tiered_sealed": 56.6951598827452,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 58.93333333333332
     },
     "score": {
      "accuracy": 0.48533333333333334,
      "calibration": 76.82347787934233,
      "cc": 26.7077945049382,
      "cc_by_type": {
       "score": 26.7077945049382
      },
      "cc_tiered": 31.28868519349913,
      "cc_tiered_open": 39.825060352839884,
      "cc_tiered_sealed": 22.752310034158377,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.7077945049382
     }
    },
    "raw_on_A": {
     "calibration": 77.62323799437446,
     "capability": 66.62869754737606,
     "intelligence": 55.63415710037767
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 58.10485941051766,
    "Emphasis on Accuracy 60:20:20": 55.076257634937356,
    "Emphasis on Cost 20:20:60": 55.417180641812784,
    "Emphasis on Speed 20:60:20": 64.81215679560749,
    "Intelligence only": 51.08240263971605
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 89.84415618532353,
    "cost": 52.6767842132323,
    "intelligence": 66.74877414880126,
    "speed": 75.19061343032087
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06831102916513103,
      "ece_score": 86.3377941669738,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 86.3377941669738,
      "score_before_time_limit": 86.3377941669738,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.12734986666666614,
      "ece": 0.0327466666666661,
      "n": 375,
      "score": 93.45066666666678,
      "score_before_time_limit": 93.45066666666678,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9306666666666666,
      "interval90_width": 0.3253333333333333,
      "n": 375,
      "nrps": 0.07579799787223554,
      "score": 89.74400772233005,
      "score_before_time_limit": 89.74400772233005,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06466092384058167
     }
    },
    "score": 89.84415618532353
   },
   "capability": 78.2964651670624,
   "composite_ci95": {
    "A": [
     66.79574834486728,
     69.343578144027
    ],
    "B": [
     65.74368259589626,
     69.59880662161049
    ],
    "C": [
     66.79574834486728,
     69.343578144027
    ]
   },
   "cost": {
    "basis": "ESTIMATE: OpenRouter google/gemma-3-12b-it public listed sibling of same Gemma family/12B size class; exact google/gemma-4-12b-it absent (endpoint404 and models-list checked2026-10-09). USD0.05/M input and0.15/M output, applied to native measured engine-basis usage; not a Gemma4 publictariff or rentalcost. Follows trusted PREP-OPEN-WEIGHTS-PROMPT Gemma4-12B fallback and v16 cygnet/decider12b precedent.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.04279556666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03779553752535497
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-12B",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.645105918311,
    "I_sealed": 62.85244237929151,
    "base": 66.74877414880126,
    "excess": 0.7045657947700086,
    "gap": 7.792663539019493,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 77.71876710209611,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 72.31377719182598,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 82.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 89.99999999999999,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 52.09577770597702,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.55742918641428,
       "hard": 53.19893501712373,
       "judge": 41.32419472837796,
       "standard": 53.816011809863554
      }
     },
     "sealed|choice": {
      "cc": 72.26629481435496,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 96.29591521761498,
       "hard": 61.508159819921204,
       "judge": 72.66514806378133,
       "standard": 81.16947472745294
      }
     },
     "sealed|noul": {
      "cc": 64.72683434402202,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 75.0,
       "hard": 60.3448275862069,
       "judge": 55.223880597014926,
       "standard": 82.6086956521739
      }
     },
     "sealed|score": {
      "cc": 51.56419797949755,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 76.8746607073365,
       "hard": 42.46762350094215,
       "judge": 55.43267392491627,
       "standard": 51.299401654560796
      }
     }
    }
   },
   "jevbench_score": 68.49882727457636,
   "key": "metask_jev_rain_12b",
   "last_measured_on": "2026-10-09",
   "listing": "wrapper",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "12deda07c573047b191b63f0768c0412120f3ed4",
   "not_ranked_because": "registry ranked=false or no composite",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.84415618532353,
     "cost": 52.6767842132323,
     "intelligence": 66.74877414880126,
     "speed": 75.9887043236542
    },
    "board_release": "v1.6.43",
    "capability": 78.2964651670624,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      66.9519161575783,
      69.51190178423195
     ],
     "B": [
      65.86465215953126,
      69.73439376375767
     ],
     "C": [
      66.9519161575783,
      69.51190178423195
     ]
    },
    "jevbench_score": 68.66306996476943,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 68.66306996476943,
     "B": 68.27147684820034,
     "C": 68.66306996476943
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.33421517265960576,
     "p50_s_raw": 0.09210758632980287,
     "p95_s_adjusted": 7.53534959163516,
     "p95_s_raw": 3.6926747958175796
    },
    "views": {
     "Balanced 33:33:33": 63.66035034197887,
     "Emphasis on Accuracy 60:20:20": 64.86077729380042,
     "Emphasis on Cost 20:20:60": 58.75959627350012,
     "Emphasis on Speed 20:60:20": 68.07834337436685,
     "Intelligence only": 66.74877414880126
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/cyankiwi/gemma-4-12B-it-qat-AWQ-INT4",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.49882727457636,
    "B": 68.14151354024378,
    "C": 68.49882727457636
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.35249755401164296,
    "p50_s_raw": 0.10124877700582147,
    "p95_s_adjusted": 8.585835219966249,
    "p95_s_raw": 4.217917609983124
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1.0,
       "cc": 96.27347996861447,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 88.82043990584336
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 96.27347996861447
      },
      "calibration_gen": {
       "accuracy": 0.7708333333333334,
       "cc": 57.06717856711694,
       "cc_by_type": {
        "choice": 58.8477366255144,
        "noul": 66.66666666666667,
        "score": 45.68713240916975
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 57.06717856711694
      },
      "composite_gen": {
       "accuracy": 0.79,
       "cc": 66.4219048681923,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 68.0,
        "score": 47.52587720620293
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 66.4219048681923
      },
      "fairness": {
       "accuracy": 0.9761904761904762,
       "cc": 93.12691462021985,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 79.38074386065956
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 93.12691462021985
      },
      "jaggedness": {
       "accuracy": 0.6419753086419753,
       "cc": 39.87106908082783,
       "cc_by_type": {
        "choice": 59.891062144095095,
        "noul": 21.42857142857142,
        "score": 38.29357366981698
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 39.87106908082783
      },
      "llm_router": {
       "accuracy": 0.8506493506493507,
       "cc": 77.84045308249219,
       "cc_by_type": {
        "choice": 75.11816641398377,
        "noul": 91.66666666666667,
        "score": 66.73652616682611
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 77.84045308249219
      },
      "long_state": {
       "accuracy": 0.14285714285714285,
       "cc": -36.80944920727243,
       "cc_by_type": {
        "choice": -12.068965517241384,
        "noul": 0.0,
        "score": -98.35938210457591
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6764705882352942,
       "cc": 54.42227412247834,
       "cc_by_type": {
        "choice": 47.36842105263158,
        "noul": 75.0,
        "score": 40.898401314803436
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 54.42227412247834
      },
      "multilingual": {
       "accuracy": 0.8140350877192982,
       "cc": 70.31326141551826,
       "cc_by_type": {
        "choice": 80.38605230386052,
        "noul": 68.29268292682926,
        "score": 62.26104901586497
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 70.31326141551826
      },
      "negation_pair_gen": {
       "accuracy": 0.7959183673469388,
       "cc": 66.23454638878256,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 52.941176470588225,
        "score": 62.429129362426124
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 66.23454638878256
      },
      "option_order": {
       "accuracy": 0.9206349206349206,
       "cc": 89.88877654196155,
       "cc_by_type": {
        "choice": 89.88877654196155
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 89.88877654196155
      },
      "perturb_pair": {
       "accuracy": 0.9193548387096774,
       "cc": 85.30050438932237,
       "cc_by_type": {
        "choice": 87.42138364779876,
        "noul": 100.0,
        "score": 68.48012952016833
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 85.30050438932237
      },
      "usecase": {
       "accuracy": 0.6862745098039216,
       "cc": 54.519610793076986,
       "cc_by_type": {
        "choice": 61.16504854368934,
        "noul": 60.00000000000001,
        "score": 42.3937838355416
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 54.519610793076986
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.7058823529411765,
       "cc": 47.2113436309134,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100.0,
        "score": -1.2231119644026789
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 47.2113436309134
      },
      "cs": {
       "accuracy": 0.7777777777777778,
       "cc": 79.3103448275862,
       "cc_by_type": {
        "choice": 58.62068965517241,
        "noul": 100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 79.3103448275862
      },
      "da": {
       "accuracy": 0.75,
       "cc": 66.81661442841975,
       "cc_by_type": {
        "choice": 68.88504753673294,
        "noul": 100.0,
        "score": 31.56479574852632
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 66.81661442841975
      },
      "de": {
       "accuracy": 0.9,
       "cc": 82.64841002900086,
       "cc_by_type": {
        "choice": 85.17908604363934,
        "noul": 81.81818181818181,
        "score": 80.9479622251814
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 82.64841002900086
      },
      "el": {
       "accuracy": 0.8235294117647058,
       "cc": 71.27641118431332,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 60.00000000000001,
        "score": 91.92447164817807
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 71.27641118431332
      },
      "en": {
       "accuracy": 0.7779735682819383,
       "cc": 64.42039924274495,
       "cc_by_type": {
        "choice": 72.52283558379663,
        "noul": 68.65671641791045,
        "score": 52.08164572652778
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 64.42039924274495
      },
      "es": {
       "accuracy": 0.8666666666666667,
       "cc": 78.93980801601477,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 63.63636363636365,
        "score": 73.18306041168066
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 78.93980801601477
      },
      "fi": {
       "accuracy": 0.7142857142857143,
       "cc": 69.69274830053051,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": 100.0,
        "score": 60.360296183642824
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 69.69274830053051
      },
      "fr": {
       "accuracy": 0.875,
       "cc": 70.65012917104805,
       "cc_by_type": {
        "choice": 85.5421686746988,
        "noul": 33.33333333333333,
        "score": 93.07488550511206
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 70.65012917104805
      },
      "hi": {
       "accuracy": 0.7307692307692307,
       "cc": 65.13808631336717,
       "cc_by_type": {
        "choice": 77.818853974122,
        "noul": 100.0,
        "score": 17.595404965979476
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 65.13808631336717
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 85.47109020402759,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 56.41327061208279
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 85.47109020402759
      },
      "ja": {
       "accuracy": 0.6818181818181818,
       "cc": 41.64421198154423,
       "cc_by_type": {
        "choice": 57.37963693764798,
        "noul": 50.0,
        "score": 17.552999006984706
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 41.64421198154423
      },
      "ko": {
       "accuracy": 0.8888888888888888,
       "cc": 90.66541975155566,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100.0,
        "score": 98.66292592133367
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 90.66541975155566
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.45235001995381,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.35705005986141
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.45235001995381
      },
      "nl": {
       "accuracy": 0.8823529411764706,
       "cc": 67.00050248305712,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 1.0015074491713571
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 67.00050248305712
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 54.68551851777729,
       "cc_by_type": {
        "choice": 84.92462311557787,
        "noul": 50.0,
        "score": 29.13193243775398
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 54.68551851777729
      },
      "pt": {
       "accuracy": 0.875,
       "cc": 77.86081066893884,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 100.0,
        "score": 58.970302246590855
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 77.86081066893884
      },
      "sv": {
       "accuracy": 0.375,
       "cc": 31.056095408809714,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": 33.33333333333333,
        "score": 53.09920159775903
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 31.056095408809714
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 40.62563994405945,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 21.876919832178356
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 40.62563994405945
      },
      "uk": {
       "accuracy": 0.7777777777777778,
       "cc": 64.70150517724299,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 60.771182198395636
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 64.70150517724299
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 13.841206507530577,
       "cc_by_type": {
        "choice": 68.08510638297872,
        "noul": 100.0,
        "score": -126.56148686038699
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 13.841206507530577
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7973333333333333,
       "cc": 73.76244270124091,
       "cc_by_type": {
        "choice": 73.76244270124091
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 73.76244270124091
      },
      "noul": {
       "accuracy": 0.8506666666666667,
       "cc": 70.13333333333334,
       "cc_by_type": {
        "noul": 70.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 70.13333333333334
      },
      "score": {
       "accuracy": 0.6906666666666667,
       "cc": 52.99097069055308,
       "cc_by_type": {
        "score": 52.99097069055308
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 52.99097069055308
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6666666666666666,
       "cc": 44.74692355042208,
       "cc_by_type": {
        "choice": 46.996466431095406,
        "noul": 50.0,
        "score": 37.24430422017083
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 44.74692355042208
      },
      "code_linting": {
       "accuracy": 0.6363636363636364,
       "cc": 39.53589630861794,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 19.999999999999996,
        "score": 17.24122929852465
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 39.53589630861794
      },
      "customer_support": {
       "accuracy": 0.7692307692307693,
       "cc": 55.53391206201687,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 66.66666666666667,
        "score": 43.41333038894915
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 55.53391206201687
      },
      "demand_forecasting": {
       "accuracy": 0.38095238095238093,
       "cc": 13.954116461127205,
       "cc_by_type": {
        "choice": 9.090909090909092,
        "noul": 50.0,
        "score": -17.22855970752748
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 13.954116461127205
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7142857142857143,
       "cc": 45.76413173276987,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 19.999999999999996,
        "score": 50.07928044421125
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 45.76413173276987
      },
      "feature_extraction": {
       "accuracy": 0.9375,
       "cc": 80.19349160347362,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 40.580474810420874
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 80.19349160347362
      },
      "financial_crime": {
       "accuracy": 0.7857142857142857,
       "cc": 79.86221446859615,
       "cc_by_type": {
        "choice": 78.72340425531915,
        "noul": 100.0,
        "score": 60.86323915046927
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 79.86221446859615
      },
      "gaming": {
       "accuracy": 0.7058823529411765,
       "cc": 73.12003051112781,
       "cc_by_type": {
        "choice": 71.89695550351288,
        "noul": 100.0,
        "score": 47.46313602987057
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 73.12003051112781
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 59.563018564812516,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0,
        "score": 41.84695043127965
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 59.563018564812516
      },
      "knowledge_graphs": {
       "accuracy": 0.5714285714285714,
       "cc": 33.961660837751445,
       "cc_by_type": {
        "choice": 46.902654867256636,
        "noul": 42.85714285714286,
        "score": 12.125184788854849
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 33.961660837751445
      },
      "lead_generation": {
       "accuracy": 0.8888888888888888,
       "cc": 69.43920550900503,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 80.0,
        "score": 28.31761652701511
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 69.43920550900503
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 65.65653183952269,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 46.96959551856804
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 65.65653183952269
      },
      "llm_guardrails": {
       "accuracy": 0.7916666666666666,
       "cc": 61.810127083374105,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 71.42857142857142,
        "score": 46.78869506745254
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 61.810127083374105
      },
      "model_routing": {
       "accuracy": 0.8817733990147784,
       "cc": 80.52579213285817,
       "cc_by_type": {
        "choice": 85.69820565450574,
        "noul": 100.0,
        "score": 55.87917074406879
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 80.52579213285817
      },
      "moderation_trust_safety": {
       "accuracy": 0.7083333333333334,
       "cc": 72.50641191707528,
       "cc_by_type": {
        "choice": 61.038961038961034,
        "noul": 100.0,
        "score": 56.4802747122648
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 72.50641191707528
      },
      "recruiting": {
       "accuracy": 0.7647058823529411,
       "cc": 65.2415200092604,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50.0,
        "score": 83.34211488671534
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 65.2415200092604
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 47.00039821586588,
       "cc_by_type": {
        "choice": 70.6601466992665,
        "noul": 33.33333333333333,
        "score": 37.0077146149978
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 47.00039821586588
      },
      "scientific_discovery": {
       "accuracy": 0.6190476190476191,
       "cc": 38.963742930892586,
       "cc_by_type": {
        "choice": 27.18446601941747,
        "noul": 50.0,
        "score": 39.70676277326029
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 38.963742930892586
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 84.49544603640166,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 53.486338109204986
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 84.49544603640166
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.9726432272681,
      90.86115546395659
     ],
     "I": [
      61.92201793536035,
      70.62816650076428
     ],
     "capability": [
      74.31683118509996,
      79.90344483944872
     ],
     "composite": {
      "A": [
       66.79574834486728,
       69.343578144027
      ],
      "B": [
       65.74368259589626,
       69.59880662161049
      ],
      "C": [
       66.79574834486728,
       69.343578144027
      ]
     },
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.84415618532353,
     "capability": 78.2964651670624,
     "intelligence": 66.74877414880126
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7973333333333333,
      "calibration": 86.3377941669738,
      "cc": 73.76244270124091,
      "cc_by_type": {
       "choice": 73.76244270124091
      },
      "cc_tiered": 74.99253095822553,
      "cc_tiered_open": 77.71876710209611,
      "cc_tiered_sealed": 72.26629481435496,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.76244270124091
     },
     "noul": {
      "accuracy": 0.8506666666666667,
      "calibration": 93.45066666666678,
      "cc": 70.13333333333334,
      "cc_by_type": {
       "noul": 70.13333333333334
      },
      "cc_tiered": 73.42380364544096,
      "cc_tiered_open": 82.1207729468599,
      "cc_tiered_sealed": 64.72683434402202,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 70.13333333333334
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 89.74400772233005,
      "cc": 52.99097069055308,
      "cc_by_type": {
       "score": 52.99097069055308
      },
      "cc_tiered": 51.82998784273728,
      "cc_tiered_open": 52.09577770597702,
      "cc_tiered_sealed": 51.56419797949755,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.99097069055308
     }
    },
    "raw_on_A": {
     "calibration": 85.98840560301178,
     "capability": 77.77430900473493,
     "intelligence": 69.56021240645808
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.47221468705639,
    "Emphasis on Accuracy 60:20:20": 64.74346385552225,
    "Emphasis on Cost 20:20:60": 58.66329878346918,
    "Emphasis on Speed 20:60:20": 67.69212046064062,
    "Intelligence only": 66.74877414880126
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Csaba Garay",
   "axes": {
    "calibration": 77.35233849488566,
    "cost": 49.35064060220574,
    "intelligence": 27.627554756423923,
    "speed": 86.3537180778988
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12841215225172864,
      "ece_score": 74.31756954965427,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 72.83121815866117,
      "score_before_time_limit": 74.31756954965427,
      "time_limit_failed_share": 0.02
     },
     "noul": {
      "abstention_rate": 0.44266666666666665,
      "brier": 0.15244666314470043,
      "ece": 0.08234546671425014,
      "n": 375,
      "score": 83.08540848831184,
      "score_before_time_limit": 83.53090665714997,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9173333333333333,
      "interval90_width": 0.4935555555555556,
      "n": 375,
      "nrps": 0.13068295844559996,
      "score": 76.14038883768394,
      "score_before_time_limit": 76.9613094720525,
      "time_limit_failed_share": 0.010666666666666666,
      "top_ece": 0.1650454260566749
     }
    },
    "score": 77.35233849488566
   },
   "capability": 52.48994662565479,
   "composite_ci95": {
    "A": [
     9.598688590352701,
     19.28097626886706
    ],
    "B": [
     8.055046474779605,
     17.013960224091093
    ],
    "C": [
     6.665755965522709,
     13.38956685337991
    ]
   },
   "cost": {
    "basis": "Estimated exact base-model public reference: Qwen/Qwen3.5-9B Darkbloom $0.08/M input and $0.13/M output, captured 2026-10-09. Provider fp4 differs from local RYOTIDE unquantized BF16; reference tariff is not measured hosting cost. Source: https://openrouter.ai/api/v1/models/qwen/qwen3.5-9b/endpoints.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.13612464
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.048787883705206224
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 21,
    "n_over_limit_answered_ok": 21,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RYOTIDE-Qwen9",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 39.61561448217045,
    "I_sealed": 18.921235479400952,
    "base": 29.268424980785703,
    "excess": 13.606281258520013,
    "gap": 20.694379002769498,
    "penalty": 0.9439371874147998,
    "per_type_split": {
     "open|choice": {
      "cc": 62.49558303248294,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 56.59012268008808,
       "judge": 68.35860250494397,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 26.282953761214635,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 30.06830665281378,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.087811240948376,
       "hard": 37.67076473034441,
       "judge": 33.89329590307768,
       "standard": 4.616154328289368
      }
     },
     "sealed|choice": {
      "cc": 47.768113114558034,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 72.21936413211235,
       "hard": 39.22341024198085,
       "judge": 49.8861047835991,
       "standard": 49.45490584737365
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 25.0,
       "hard": -24.13793103448276,
       "judge": -16.417910447761198,
       "standard": 13.043478260869556
      }
     },
     "sealed|score": {
      "cc": 8.995593323644819,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 35.843219543729575,
       "hard": 1.8186910992705863,
       "judge": -2.3511609711775794,
       "standard": 26.945716104584506
      }
     }
    }
   },
   "jevbench_score": 14.694072442614042,
   "key": "ryotide_qwen9",
   "last_measured_on": "2026-10-10",
   "licence": "MIT engine on Apache-2.0 weights",
   "listing": "ranked",
   "listing_note": "Reclassified 12 Oct 2026 from wrapper to native ranked: a training-free typed-decision readout over unmodified open weights (Qwen/Qwen3.5-9B, revision c2022362), like the other native open-weights systems, not a service running another entrant's model. v1.6.4 scores, axes and cost unchanged; it was never part of the four-native v1.6.3 field median and does not enter it.",
   "measured_in": "v1.6.4",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.2699285596189,
     "cost": 49.35064060220574,
     "intelligence": 28.676598568644657,
     "speed": 86.71260028928383
    },
    "board_release": "v1.6.43",
    "capability": 53.473263564131784,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 88,
    "capability_rank": 114,
    "composite_ci95": {
     "A": [
      10.301952384081664,
      20.49332468149985
     ],
     "B": [
      8.686711300573288,
      18.11978988063779
     ],
     "C": [
      7.154133600056712,
      14.23147547326378
     ]
    },
    "jevbench_score": 16.1347687863636,
    "measured_in": "v1.6.4",
    "method": "v1.6",
    "open_board_rank": 115,
    "rank": 153,
    "ranks": {
     "A": 153,
     "B": 153,
     "C": 153,
     "capability": 169
    },
    "scores": {
     "A": 16.1347687863636,
     "B": 14.016103437178304,
     "C": 11.204700546085835
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4132151529999192,
     "p50_s_raw": 0.13160757649995958,
     "p95_s_adjusted": 0.5158978665999938,
     "p95_s_raw": 0.18294893329999692
    },
    "views": {
     "Balanced 33:33:33": 14.420164341278662,
     "Emphasis on Accuracy 60:20:20": 11.745818244481526,
     "Emphasis on Cost 20:20:60": 14.947291611405522,
     "Emphasis on Speed 20:60:20": 17.856002324382263,
     "Intelligence only": 9.189428059162678
    }
   },
   "rank": 179,
   "ranked": true,
   "ranks": {
    "A": 179,
    "B": 180,
    "C": 181,
    "capability": 189
   },
   "repo": "https://github.com/csabag/ryotide/tree/94c71a3d9f044ffaba87981e985b66f47de3be80",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 14.694072442614042,
    "B": 12.693189454342397,
    "C": 10.204216974037529
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.41632815200002826,
    "p50_s_raw": 0.13316407600001412,
    "p95_s_adjusted": 0.5561506042999099,
    "p95_s_raw": 0.20307530214995495
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.76,
       "cc": 66.05136415304769,
       "cc_by_type": {
        "choice": 64.44444444444444,
        "noul": 50.0,
        "score": 83.70964801469863
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 66.05136415304769
      },
      "calibration_gen": {
       "accuracy": 0.4479166666666667,
       "cc": 14.44148530317438,
       "cc_by_type": {
        "choice": 42.38683127572016,
        "noul": 33.33333333333333,
        "score": -32.395708699530346
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 14.44148530317438
      },
      "composite_gen": {
       "accuracy": 0.37,
       "cc": -19.57376093365861,
       "cc_by_type": {
        "choice": 47.96747967479675,
        "noul": -68.0,
        "score": -38.68876247577258
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.7142857142857143,
       "cc": 71.76642296515836,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100.0,
        "score": 35.63825194632254
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 71.76642296515836
      },
      "jaggedness": {
       "accuracy": 0.3888888888888889,
       "cc": -12.593176711081462,
       "cc_by_type": {
        "choice": 33.15177024015846,
        "noul": -85.71428571428572,
        "score": 14.782985340882881
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.7207792207792207,
       "cc": 56.929464534321276,
       "cc_by_type": {
        "choice": 59.86801034513512,
        "noul": 52.08333333333333,
        "score": 58.83704992449537
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 56.929464534321276
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -131.99233716475095,
       "cc_by_type": {
        "choice": -29.310344827586214,
        "noul": -100.0,
        "score": -266.66666666666663
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4411764705882353,
       "cc": 0.9235219934698241,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -50.0,
        "score": 15.92846071725158
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 0.9235219934698241
      },
      "multilingual": {
       "accuracy": 0.5859649122807018,
       "cc": 32.2795517679191,
       "cc_by_type": {
        "choice": 50.49813200498132,
        "noul": 9.756097560975618,
        "score": 36.584425737800366
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 32.2795517679191
      },
      "negation_pair_gen": {
       "accuracy": 0.3979591836734694,
       "cc": -11.709890170422623,
       "cc_by_type": {
        "choice": 43.33333333333332,
        "noul": -41.17647058823529,
        "score": -37.28653325636591
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.6666666666666666,
       "cc": 57.532861476238615,
       "cc_by_type": {
        "choice": 57.532861476238615
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 57.532861476238615
      },
      "perturb_pair": {
       "accuracy": 0.7419354838709677,
       "cc": 45.73965152932314,
       "cc_by_type": {
        "choice": 71.69811320754718,
        "noul": 16.666666666666675,
        "score": 48.85417471375556
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 45.73965152932314
      },
      "usecase": {
       "accuracy": 0.46078431372549017,
       "cc": 14.920533192964271,
       "cc_by_type": {
        "choice": 52.203136669156095,
        "noul": -24.0,
        "score": 16.558462909736715
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 14.920533192964271
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.47058823529411764,
       "cc": 15.197907909052608,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": 2.7365808700149707
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 15.197907909052608
      },
      "cs": {
       "accuracy": 0.7777777777777778,
       "cc": 56.32183908045977,
       "cc_by_type": {
        "choice": 79.3103448275862,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 56.32183908045977
      },
      "da": {
       "accuracy": 0.6875,
       "cc": 14.572239106386718,
       "cc_by_type": {
        "choice": 84.44252376836647,
        "noul": 0.0,
        "score": -40.72580644920632
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 14.572239106386718
      },
      "de": {
       "accuracy": 0.7,
       "cc": 60.32572841250546,
       "cc_by_type": {
        "choice": 55.53725813091807,
        "noul": 63.63636363636365,
        "score": 61.80356347023468
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 60.32572841250546
      },
      "el": {
       "accuracy": 0.5294117647058824,
       "cc": 27.556367507464582,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -19.999999999999996,
        "score": 59.81195966525089
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 27.556367507464582
      },
      "en": {
       "accuracy": 0.5259911894273128,
       "cc": 18.612787223400975,
       "cc_by_type": {
        "choice": 50.18367752184269,
        "noul": -5.223880597014929,
        "score": 10.878564745375174
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 18.612787223400975
      },
      "es": {
       "accuracy": 0.8333333333333334,
       "cc": 69.61549652183463,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 63.63636363636365,
        "score": 45.21012592914022
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 69.61549652183463
      },
      "fi": {
       "accuracy": 0.42857142857142855,
       "cc": -0.0686132800628864,
       "cc_by_type": {
        "choice": 23.076923076923077,
        "noul": -100.0,
        "score": 76.71723708288826
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5625,
       "cc": 12.535202989271738,
       "cc_by_type": {
        "choice": 42.16867469879518,
        "noul": -100.0,
        "score": 95.43693426902003
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 12.535202989271738
      },
      "hi": {
       "accuracy": 0.5,
       "cc": 1.3565370500971834,
       "cc_by_type": {
        "choice": 44.547134935304996,
        "noul": -33.333333333333336,
        "score": -7.14419045168011
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 1.3565370500971834
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.7692307692307693,
       "cc": 55.70093751575863,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 33.33333333333333,
        "score": 49.78371408938743
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 55.70093751575863
      },
      "ja": {
       "accuracy": 0.45454545454545453,
       "cc": -6.061268823867529,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 0.0,
        "score": -61.35665572179991
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.4444444444444444,
       "cc": -4.546760795957428,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -100.0,
        "score": 39.69305094546105
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9333333333333333,
       "cc": 91.97613585457272,
       "cc_by_type": {
        "choice": 77.77777777777779,
        "noul": 100.0,
        "score": 98.1506297859404
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 91.97613585457272
      },
      "nl": {
       "accuracy": 0.4117647058823529,
       "cc": -1.0752965494261122,
       "cc_by_type": {
        "choice": 13.738019169329075,
        "noul": 0.0,
        "score": -16.96390881760741
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 29.411861798854904,
       "cc_by_type": {
        "choice": 69.84924623115577,
        "noul": 16.666666666666675,
        "score": 1.719672498742264
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 29.411861798854904
      },
      "pt": {
       "accuracy": 0.5833333333333334,
       "cc": 44.85538134603173,
       "cc_by_type": {
        "choice": 36.530324400564176,
        "noul": 50.0,
        "score": 48.03581963753102
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 44.85538134603173
      },
      "sv": {
       "accuracy": 0.25,
       "cc": -2.8689134454598872,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": -33.333333333333336,
        "score": 17.990841701616887
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 8.662553165653643,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": -30.534079633473855
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 8.662553165653643
      },
      "uk": {
       "accuracy": 0.6666666666666666,
       "cc": 50.95552616234304,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 84.04937418595387
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 50.95552616234304
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -18.2554748281775,
       "cc_by_type": {
        "choice": 36.170212765957444,
        "noul": 33.33333333333333,
        "score": -124.26997058382328
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.62,
       "cc": 50.80458006482671,
       "cc_by_type": {
        "choice": 50.80458006482671
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.80458006482671
      },
      "noul": {
       "accuracy": 0.5066666666666667,
       "cc": 1.333333333333342,
       "cc_by_type": {
        "noul": 1.333333333333342
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 1.333333333333342
      },
      "score": {
       "accuracy": 0.432,
       "cc": 16.638727413057563,
       "cc_by_type": {
        "score": 16.638727413057563
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 16.638727413057563
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4,
       "cc": -0.33698217915645756,
       "cc_by_type": {
        "choice": 46.996466431095406,
        "noul": -50.0,
        "score": 1.9925870314352223
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4090909090909091,
       "cc": 6.182899746566381,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -19.999999999999996,
        "score": -5.550679642288436
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 6.182899746566381
      },
      "customer_support": {
       "accuracy": 0.5384615384615384,
       "cc": 22.1821554104641,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0.0,
        "score": 31.763857535740126
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 22.1821554104641
      },
      "demand_forecasting": {
       "accuracy": 0.2857142857142857,
       "cc": -30.31949857431742,
       "cc_by_type": {
        "choice": 9.090909090909092,
        "noul": -100.0,
        "score": -0.049404813861353425
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 25.651052971843274,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 19.999999999999996,
        "score": 22.526929407333107
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 25.651052971843274
      },
      "feature_extraction": {
       "accuracy": 0.4375,
       "cc": -20.811031444586995,
       "cc_by_type": {
        "choice": 21.875,
        "noul": -100.0,
        "score": 15.691905666239014
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 36.86432069029883,
       "cc_by_type": {
        "choice": 57.44680851063829,
        "noul": 19.999999999999996,
        "score": 33.14615356025822
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 36.86432069029883
      },
      "gaming": {
       "accuracy": 0.29411764705882354,
       "cc": -24.720180964179843,
       "cc_by_type": {
        "choice": 15.69086651053864,
        "noul": -100.0,
        "score": 10.148590596921835
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 18.69117622868764,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 0.0,
        "score": 35.02089710711556
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 18.69117622868764
      },
      "knowledge_graphs": {
       "accuracy": 0.3333333333333333,
       "cc": -1.336281990839917,
       "cc_by_type": {
        "choice": 20.353982300884958,
        "noul": -42.85714285714286,
        "score": 18.49431458373815
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.5925925925925926,
       "cc": 34.86130124669457,
       "cc_by_type": {
        "choice": 79.79797979797979,
        "noul": 0.0,
        "score": 24.785923942103917
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 34.86130124669457
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 5.407718729482173,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -75.0,
        "score": -8.77684381155348
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 5.407718729482173
      },
      "llm_guardrails": {
       "accuracy": 0.5416666666666666,
       "cc": 21.13264329944355,
       "cc_by_type": {
        "choice": 50.81967213114754,
        "noul": -14.28571428571429,
        "score": 26.8639720528974
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 21.13264329944355
      },
      "model_routing": {
       "accuracy": 0.7881773399014779,
       "cc": 64.2670663183282,
       "cc_by_type": {
        "choice": 73.78004369992718,
        "noul": 76.27118644067797,
        "score": 42.74996881437948
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 64.2670663183282
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 60.10650947754033,
       "cc_by_type": {
        "choice": 41.55844155844156,
        "noul": 66.66666666666667,
        "score": 72.09442020751275
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 60.10650947754033
      },
      "recruiting": {
       "accuracy": 0.5294117647058824,
       "cc": 13.055323558022193,
       "cc_by_type": {
        "choice": 81.19122257053293,
        "noul": -50.0,
        "score": 7.9747481035336465
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 13.055323558022193
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 5.36471360451591,
       "cc_by_type": {
        "choice": 55.99022004889975,
        "noul": -33.333333333333336,
        "score": -6.562745902018685
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 5.36471360451591
      },
      "scientific_discovery": {
       "accuracy": 0.38095238095238093,
       "cc": -0.6081023002725369,
       "cc_by_type": {
        "choice": 27.18446601941747,
        "noul": -50.0,
        "score": 20.99122707976492
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 46.972150398582045,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": -19.08354880425387
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 46.972150398582045
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.06512048464919,
      79.83352815497564
     ],
     "I": [
      23.25648858171055,
      30.89797430499399
     ],
     "capability": [
      48.41837842556112,
      54.43007105303262
     ],
     "composite": {
      "A": [
       9.598688590352701,
       19.28097626886706
      ],
      "B": [
       8.055046474779605,
       17.013960224091093
      ],
      "C": [
       6.665755965522709,
       13.38956685337991
      ]
     },
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.35233849488566,
     "capability": 52.48994662565479,
     "intelligence": 27.627554756423923
    },
    "per_type": {
     "choice": {
      "accuracy": 0.62,
      "calibration": 72.83121815866117,
      "cc": 50.80458006482671,
      "cc_by_type": {
       "choice": 50.80458006482671
      },
      "cc_tiered": 55.13184807352049,
      "cc_tiered_open": 62.49558303248294,
      "cc_tiered_sealed": 47.768113114558034,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.80458006482671
     },
     "noul": {
      "accuracy": 0.5066666666666667,
      "calibration": 83.08540848831184,
      "cc": 1.333333333333342,
      "cc_by_type": {
       "noul": 1.333333333333342
      },
      "cc_tiered": 13.141476880607318,
      "cc_tiered_open": 26.282953761214635,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 1.333333333333342
     },
     "score": {
      "accuracy": 0.432,
      "calibration": 76.14038883768394,
      "cc": 16.638727413057563,
      "cc_by_type": {
       "score": 16.638727413057563
      },
      "cc_tiered": 19.5319499882293,
      "cc_tiered_open": 30.06830665281378,
      "cc_tiered_sealed": 8.995593323644819,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 16.638727413057563
     }
    },
    "raw_on_A": {
     "calibration": 77.84867467337358,
     "capability": 54.81455349282751,
     "intelligence": 31.78043231228143
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.020000000000000018,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 13.114526381662106,
    "Emphasis on Accuracy 60:20:20": 10.590061165323435,
    "Emphasis on Cost 20:20:60": 13.698360714218317,
    "Emphasis on Speed 20:60:20": 16.30672952176207,
    "Intelligence only": 8.21737140705745
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 78.10292780844634,
    "cost": 54.30361484501323,
    "intelligence": 41.00304543028662,
    "speed": 82.31428682922845
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12842390280324772,
      "ece_score": 74.31521943935046,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 73.62161072458319,
      "score_before_time_limit": 74.31521943935046,
      "time_limit_failed_share": 0.009333333333333334
     },
     "noul": {
      "abstention_rate": 0.33333333333333337,
      "brier": 0.11388917204323451,
      "ece": 0.06372845741047697,
      "n": 375,
      "score": 87.2543085179046,
      "score_before_time_limit": 87.2543085179046,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8053333333333333,
      "interval90_width": 0.24711111111111111,
      "n": 375,
      "nrps": 0.10762527231724621,
      "score": 73.43286418285125,
      "score_before_time_limit": 73.6292087394899,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.20989527644647793
     }
    },
    "score": 78.10292780844634
   },
   "capability": 59.55298661936648,
   "composite_ci95": {
    "A": [
     27.437436134100544,
     52.43739882732885
    ],
    "B": [
     24.601071659349397,
     49.19356398798599
    ],
    "C": [
     19.05377509312538,
     36.414860296756146
    ]
   },
   "cost": {
    "basis": "Self-hosted open weights, no bookable tariff. v1.5-M2 base-model reference rule: base model Qwen/Qwen3.5-4B (model card base_model) -> DeepInfra Qwen/Qwen3.5-4B, USD 0.03/M input and USD 0.15/M output, frozen 25 Sep 2026 price cut-off snapshot (METHOD-v1.5-ADDENDUM-PRICING-INTERPRETATION-1 I-2/I-3; listing deprecated after the cut-off, stays the frozen v1.5 reference). Output tokens priced at the reference output price. Read 2026-10-08.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.12006908000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03335898580121703
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 8,
    "n_over_limit_answered_ok": 8,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Wald 4B",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.50880508565272,
    "I_sealed": 39.49728577492053,
    "base": 41.00304543028662,
    "excess": -4.0765784335172945,
    "gap": 3.01151931073219,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.2690514819802,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 66.02705253224286,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 16.316770186335404,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 40.94059358864256,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.042005572750426,
       "hard": 44.3073169155134,
       "judge": 46.64410687719149,
       "standard": 19.601171010023556
      }
     },
     "sealed|choice": {
      "cc": 63.21007145303903,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 83.3316184792674,
       "hard": 56.443444006752955,
       "judge": 60.82004555808657,
       "standard": 70.26759167492567
      }
     },
     "sealed|noul": {
      "cc": 18.474093177291948,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 54.16666666666667,
       "hard": 15.517241379310342,
       "judge": 4.477611940298498,
       "standard": 27.536231884057962
      }
     },
     "sealed|score": {
      "cc": 36.8076926944306,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 58.68738567055003,
       "hard": 33.084604932395,
       "judge": 27.44752759181396,
       "standard": 47.35426938436701
      }
     }
    }
   },
   "jevbench_score": 39.70151539139312,
   "key": "wald-4b-v2",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "919a9d72a65379271005ed0945bfc72742dad3af",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.39957889891498,
     "cost": 54.30361484501323,
     "intelligence": 41.11158964477434,
     "speed": 82.20577865115169
    },
    "board_release": "v1.6.43",
    "capability": 59.75558427184466,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 61,
    "capability_rank": 86,
    "composite_ci95": {
     "A": [
      27.711939304816244,
      52.55504034148649
     ],
     "B": [
      24.89942588247082,
      49.38746598254962
     ],
     "C": [
      19.244402295011277,
      36.49655579269895
     ]
    },
    "jevbench_score": 39.9690949735785,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": 54,
    "rank": 83,
    "ranks": {
     "A": 83,
     "B": 84,
     "C": 85,
     "capability": 133
    },
    "scores": {
     "A": 39.9690949735785,
     "B": 36.74952891978881,
     "C": 27.756315953873955
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21244139187037944,
     "p50_s_raw": 0.031220695935189724,
     "p95_s_adjusted": 2.8325852981768525,
     "p95_s_raw": 1.3412926490884263
    },
    "views": {
     "Balanced 33:33:33": 36.94101579201056,
     "Emphasis on Accuracy 60:20:20": 32.64384498780741,
     "Emphasis on Cost 20:20:60": 36.849381766766875,
     "Emphasis on Speed 20:60:20": 42.663206822382165,
     "Intelligence only": 27.7941118378976
    }
   },
   "rank": 95,
   "ranked": true,
   "ranks": {
    "A": 95,
    "B": 96,
    "C": 98,
    "capability": 153
   },
   "repo": "https://huggingface.co/org2ai/Wald-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 39.70151539139312,
    "B": 36.49178107306264,
    "C": 27.570496799578564
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23851143810898065,
    "p50_s_raw": 0.04425571905449033,
    "p95_s_adjusted": 2.460719473008066,
    "p95_s_raw": 1.1553597365040331
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.72,
       "cc": 68.2119055157796,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 50.0,
        "score": 99.08016099178325
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 68.2119055157796
      },
      "calibration_gen": {
       "accuracy": 0.7395833333333334,
       "cc": 43.20138852122932,
       "cc_by_type": {
        "choice": 75.30864197530865,
        "noul": 25.0,
        "score": 29.295523588379304
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 43.20138852122932
      },
      "composite_gen": {
       "accuracy": 0.61,
       "cc": 27.594758393773663,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": -12.0,
        "score": 20.80053534392261
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 27.594758393773663
      },
      "fairness": {
       "accuracy": 0.8333333333333334,
       "cc": 81.41361209300184,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100.0,
        "score": 64.579819329853
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 81.41361209300184
      },
      "jaggedness": {
       "accuracy": 0.5493827160493827,
       "cc": 8.823790192360185,
       "cc_by_type": {
        "choice": 53.94899727655361,
        "noul": -42.85714285714286,
        "score": 15.379516157669803
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 8.823790192360185
      },
      "llm_router": {
       "accuracy": 0.775974025974026,
       "cc": 64.44463445590775,
       "cc_by_type": {
        "choice": 69.49968786230271,
        "noul": 70.83333333333333,
        "score": 53.00088217208718
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 64.44463445590775
      },
      "long_state": {
       "accuracy": 0.2857142857142857,
       "cc": -27.03945295275031,
       "cc_by_type": {
        "choice": 13.793103448275854,
        "noul": 0.0,
        "score": -94.91146230652679
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5,
       "cc": 7.403242931791567,
       "cc_by_type": {
        "choice": 47.36842105263158,
        "noul": -75.0,
        "score": 49.841307742743126
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 7.403242931791567
      },
      "multilingual": {
       "accuracy": 0.6491228070175439,
       "cc": 38.093835945068015,
       "cc_by_type": {
        "choice": 63.57409713574097,
        "noul": 9.756097560975618,
        "score": 40.95131313848746
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 38.093835945068015
      },
      "negation_pair_gen": {
       "accuracy": 0.5204081632653061,
       "cc": 20.521786356237072,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": -29.411764705882348,
        "score": 30.977123774593572
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 20.521786356237072
      },
      "option_order": {
       "accuracy": 0.873015873015873,
       "cc": 83.82204246713852,
       "cc_by_type": {
        "choice": 83.82204246713852
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 83.82204246713852
      },
      "perturb_pair": {
       "accuracy": 0.8064516129032258,
       "cc": 72.61751779733824,
       "cc_by_type": {
        "choice": 71.69811320754718,
        "noul": 83.33333333333333,
        "score": 62.821106851134225
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 72.61751779733824
      },
      "usecase": {
       "accuracy": 0.6274509803921569,
       "cc": 42.32708851945942,
       "cc_by_type": {
        "choice": 61.16504854368934,
        "noul": 19.999999999999996,
        "score": 45.81621701468893
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 42.32708851945942
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.47058823529411764,
       "cc": 14.120375284224684,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0.0,
        "score": 18.55160204315024
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 14.120375284224684
      },
      "cs": {
       "accuracy": 0.6666666666666666,
       "cc": 68.9655172413793,
       "cc_by_type": {
        "choice": 37.93103448275862,
        "noul": 100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 68.9655172413793
      },
      "da": {
       "accuracy": 0.6875,
       "cc": 50.49501668418102,
       "cc_by_type": {
        "choice": 68.88504753673294,
        "noul": 50.0,
        "score": 32.60000251581009
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 50.49501668418102
      },
      "de": {
       "accuracy": 0.725,
       "cc": 54.187380432772635,
       "cc_by_type": {
        "choice": 70.3581720872787,
        "noul": 27.27272727272727,
        "score": 64.93124193831193
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 54.187380432772635
      },
      "el": {
       "accuracy": 0.47058823529411764,
       "cc": 15.393732253336765,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -60.0,
        "score": 63.32405390286744
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 15.393732253336765
      },
      "en": {
       "accuracy": 0.6757709251101321,
       "cc": 43.45231827001086,
       "cc_by_type": {
        "choice": 65.82108816521045,
        "noul": 24.626865671641784,
        "score": 39.90900097318036
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 43.45231827001086
      },
      "es": {
       "accuracy": 0.7333333333333333,
       "cc": 51.07663096036739,
       "cc_by_type": {
        "choice": 75.9197324414716,
        "noul": 27.27272727272727,
        "score": 50.03743316690328
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 51.07663096036739
      },
      "fi": {
       "accuracy": 0.42857142857142855,
       "cc": 1.9662469414521173,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": -100.0,
        "score": 57.180792106407644
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 1.9662469414521173
      },
      "fr": {
       "accuracy": 0.8125,
       "cc": 47.283240126339756,
       "cc_by_type": {
        "choice": 85.5421686746988,
        "noul": -33.333333333333336,
        "score": 89.64088503765379
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 47.283240126339756
      },
      "hi": {
       "accuracy": 0.6153846153846154,
       "cc": 26.24706077168248,
       "cc_by_type": {
        "choice": 77.818853974122,
        "noul": -33.333333333333336,
        "score": 34.255661674258775
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 26.24706077168248
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 76.36299722208126,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 95.75565833291049
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 76.36299722208126
      },
      "ja": {
       "accuracy": 0.5,
       "cc": -10.117201418040365,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 25.0,
        "score": -98.52445350431842
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.6666666666666666,
       "cc": 74.45318709325632,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100.0,
        "score": 50.02622794643561
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 74.45318709325632
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 95.32198614555834,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 85.96595843667502
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 95.32198614555834
      },
      "nl": {
       "accuracy": 0.7058823529411765,
       "cc": 8.120626845785097,
       "cc_by_type": {
        "choice": 71.24600638977637,
        "noul": 0.0,
        "score": -46.88412585242108
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 8.120626845785097
      },
      "no": {
       "accuracy": 0.25,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 35.48908488681278,
       "cc_by_type": {
        "choice": 77.38693467336684,
        "noul": 16.666666666666675,
        "score": 12.41365332040485
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 35.48908488681278
      },
      "pt": {
       "accuracy": 0.7083333333333334,
       "cc": 52.32917576828053,
       "cc_by_type": {
        "choice": 61.918194640338506,
        "noul": 50.0,
        "score": 45.0693326645031
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 52.32917576828053
      },
      "sv": {
       "accuracy": 0.25,
       "cc": 7.519324033911375,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": -33.333333333333336,
        "score": 49.155554139730675
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 7.519324033911375
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 76.85175629288626,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": 74.033529748224
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 76.85175629288626
      },
      "uk": {
       "accuracy": 0.4444444444444444,
       "cc": 12.647260678131401,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": 35.7912443999856
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 12.647260678131401
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -13.600416695423213,
       "cc_by_type": {
        "choice": 68.08510638297872,
        "noul": -33.333333333333336,
        "score": -75.55302313591503
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7333333333333333,
       "cc": 65.47689829110645,
       "cc_by_type": {
        "choice": 65.47689829110645
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.47689829110645
      },
      "noul": {
       "accuracy": 0.608,
       "cc": 21.599999999999998,
       "cc_by_type": {
        "noul": 21.599999999999998
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 21.599999999999998
      },
      "score": {
       "accuracy": 0.6053333333333333,
       "cc": 40.021737739599615,
       "cc_by_type": {
        "score": 40.021737739599615
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 40.021737739599615
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 18.513631745529423,
       "cc_by_type": {
        "choice": 36.39575971731449,
        "noul": 0.0,
        "score": 19.145135519273783
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 18.513631745529423
      },
      "code_linting": {
       "accuracy": 0.5454545454545454,
       "cc": 11.54303802688237,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -60.0,
        "score": 50.52973519865953
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 11.54303802688237
      },
      "customer_support": {
       "accuracy": 0.38461538461538464,
       "cc": -26.722943576168863,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": -70.0239031922747
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.5238095238095238,
       "cc": 10.709697254585473,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 0.0,
        "score": -15.922856288191634
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 10.709697254585473
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6428571428571429,
       "cc": 38.283456663102605,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 19.999999999999996,
        "score": 60.42414048111109
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 38.283456663102605
      },
      "feature_extraction": {
       "accuracy": 0.625,
       "cc": 14.896396249774858,
       "cc_by_type": {
        "choice": 53.125,
        "noul": 33.33333333333333,
        "score": -41.76914458400876
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 14.896396249774858
      },
      "financial_crime": {
       "accuracy": 0.5714285714285714,
       "cc": 38.56767797990075,
       "cc_by_type": {
        "choice": 57.44680851063829,
        "noul": 19.999999999999996,
        "score": 38.25622542906396
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 38.56767797990075
      },
      "gaming": {
       "accuracy": 0.6470588235294118,
       "cc": 72.00065469480536,
       "cc_by_type": {
        "choice": 71.89695550351288,
        "noul": 100.0,
        "score": 44.10500858090322
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 72.00065469480536
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 21.988315835975936,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 0.0,
        "score": 13.333368560559379
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 21.988315835975936
      },
      "knowledge_graphs": {
       "accuracy": 0.3333333333333333,
       "cc": -1.437337178217594,
       "cc_by_type": {
        "choice": 20.353982300884958,
        "noul": -42.85714285714286,
        "score": 18.191149021605124
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.7777777777777778,
       "cc": 63.372521556256174,
       "cc_by_type": {
        "choice": 89.89898989898991,
        "noul": 19.999999999999996,
        "score": 80.21857476977863
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 63.372521556256174
      },
      "legal_compliance": {
       "accuracy": 0.5333333333333333,
       "cc": 30.51519933101532,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -50.0,
        "score": 41.545597993045966
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 30.51519933101532
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 42.73737839090273,
       "cc_by_type": {
        "choice": 83.60655737704917,
        "noul": 14.28571428571428,
        "score": 30.31986350994473
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 42.73737839090273
      },
      "model_routing": {
       "accuracy": 0.7931034482758621,
       "cc": 65.46757086712965,
       "cc_by_type": {
        "choice": 78.5473084817586,
        "noul": 76.27118644067797,
        "score": 41.58421767895237
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 65.46757086712965
      },
      "moderation_trust_safety": {
       "accuracy": 0.5833333333333334,
       "cc": 42.13343102692523,
       "cc_by_type": {
        "choice": 61.038961038961034,
        "noul": 33.33333333333333,
        "score": 32.02799870848132
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 42.13343102692523
      },
      "recruiting": {
       "accuracy": 0.7647058823529411,
       "cc": 66.50324086313627,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50.0,
        "score": 87.12727744834298
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 66.50324086313627
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 32.07478294962468,
       "cc_by_type": {
        "choice": 26.650366748166256,
        "noul": 33.33333333333333,
        "score": 36.24064876737445
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 32.07478294962468
      },
      "scientific_discovery": {
       "accuracy": 0.47619047619047616,
       "cc": 11.88079162408494,
       "cc_by_type": {
        "choice": 27.18446601941747,
        "noul": -50.0,
        "score": 58.45790885283735
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 11.88079162408494
      },
      "search_retrieval": {
       "accuracy": 0.7916666666666666,
       "cc": 66.28711066326191,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 38.86133198978572
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 66.28711066326191
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.65514052121753,
      80.28072103879629
     ],
     "I": [
      35.14753958034039,
      46.18595337580061
     ],
     "capability": [
      54.830282728287756,
      62.33343067078398
     ],
     "composite": {
      "A": [
       27.437436134100544,
       52.43739882732885
      ],
      "B": [
       24.601071659349397,
       49.19356398798599
      ],
      "C": [
       19.05377509312538,
       36.414860296756146
      ]
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 78.10292780844634,
     "capability": 59.55298661936648,
     "intelligence": 41.00304543028662
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 73.62161072458319,
      "cc": 65.47689829110645,
      "cc_by_type": {
       "choice": 65.47689829110645
      },
      "cc_tiered": 66.73956146750962,
      "cc_tiered_open": 70.2690514819802,
      "cc_tiered_sealed": 63.21007145303903,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.47689829110645
     },
     "noul": {
      "accuracy": 0.608,
      "calibration": 87.2543085179046,
      "cc": 21.599999999999998,
      "cc_by_type": {
       "noul": 21.599999999999998
      },
      "cc_tiered": 17.395431681813676,
      "cc_tiered_open": 16.316770186335404,
      "cc_tiered_sealed": 18.474093177291948,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 21.599999999999998
     },
     "score": {
      "accuracy": 0.6053333333333333,
      "calibration": 73.43286418285125,
      "cc": 40.021737739599615,
      "cc_by_type": {
       "score": 40.021737739599615
      },
      "cc_tiered": 38.87414314153658,
      "cc_tiered_open": 40.94059358864256,
      "cc_tiered_sealed": 36.8076926944306,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 40.021737739599615
     }
    },
    "raw_on_A": {
     "calibration": 73.5325444400666,
     "capability": 59.23878738188296,
     "intelligence": 44.945030323699314
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.009333333333333305,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 36.71387223389962,
    "Emphasis on Accuracy 60:20:20": 32.41624579352294,
    "Emphasis on Cost 20:20:60": 36.63574483730573,
    "Emphasis on Speed 20:60:20": 42.429502161290515,
    "Intelligence only": 27.574543698298026
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Pixilab",
   "axes": {
    "calibration": 90.67498392881862,
    "cost": 49.62845783533821,
    "intelligence": 49.21581563112197,
    "speed": 91.13697610522254
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04822184846568736,
      "ece_score": 90.35563030686254,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 90.35563030686254,
      "score_before_time_limit": 90.35563030686254,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3296703296703297,
      "brier": 0.10252744982113196,
      "ece": 0.049136862184262595,
      "n": 364,
      "score": 90.17262756314747,
      "score_before_time_limit": 90.17262756314747,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9483695652173914,
      "interval90_width": 0.3299365942028985,
      "n": 368,
      "nrps": 0.07594938571288598,
      "score": 91.49669391644588,
      "score_before_time_limit": 91.49669391644588,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.047058367979098334
     }
    },
    "score": 90.67498392881862
   },
   "capability": 69.9453997799703,
   "composite_ci95": {
    "A": [
     45.20260615882145,
     64.73830080500183
    ],
    "B": [
     41.81467673642534,
     62.04610921006097
    ],
    "C": [
     31.390698721403783,
     52.4463580470654
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model reference. v1.5-M2 base-model reference: Blink v0.3 26B-A4B is a full fine-tune of google/gemma-4-26B-A4B-it quantised to NVFP4 (ModelOpt); the base resolves in pricing_v15 as base_model google/gemma-4-26b-a4b-it -> OpenRouter google/gemma-4-26b-a4b-it, USD 0.09/M input, USD 0.30/M output, from the OpenRouter catalog snapshot 2026-09-25 (jevbench-v15-measure-20260925/receipts/OPENROUTER-MODELS-20260925.json, sha256 f5d0cd8e0f7d2d6514748c99bfcaeacc09775bac7aa7eecf765af4ba04b0dcd5), read by the frozen pricing_v15 on 2026-10-10. Tokens: input = vLLM prompt_tokens, output = 1 per answered question (one decode step, the option letter); both priced. The 29 context-refused decisions carry no tokens and stay in the 1,500-decision denominator.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04775858
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Blink v0.3 26B-A4B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX 5090 32 GB pod (Drobeta-Turnu Severin, Romania), offline after setup (egress cut and proven), plain vLLM 0.30.0 with the model card's own serve command plus ONE serving-only deviation: VLLM_BATCH_INVARIANT=1 (needed to pass the 0.005 repeatability gate; see DETERMINISM.md), served through our loopback shim implementing the card's reference decide() client (T=1.3, softmax over option-letter logprobs); context 32,768 (29 of 1,500 states are 77k-82k tokens and were refused by vLLM = recorded misses)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 53.29629960522241,
    "I_sealed": 45.135331657021524,
    "base": 49.21581563112197,
    "excess": -1.9122766331388288,
    "gap": 8.160967948200884,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.01883434266433,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.0,
       "hard": 67.9144385026738,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 28.205659075224297,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 57.66440539777861,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.90386806087209,
       "hard": 59.08254746680842,
       "judge": 54.11814783561007,
       "standard": 55.527776271425026
      }
     },
     "sealed|choice": {
      "cc": 68.42011936132744,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 83.10564188132234,
       "hard": 59.93184566340551,
       "judge": 68.95134696362436,
       "standard": 77.25706409372847
      }
     },
     "sealed|noul": {
      "cc": 20.840080971659916,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 64.1025641025641,
       "hard": 0.0,
       "judge": 30.555555555555557,
       "standard": 26.315789473684205
      }
     },
     "sealed|score": {
      "cc": 46.145794638077234,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 59.269920865495806,
       "hard": 31.19109112767333,
       "judge": 57.63492575992532,
       "standard": 52.25944186240362
      }
     }
    }
   },
   "jevbench_score": 61.12013368419001,
   "key": "blink-v0-3-26b-a4b-nvfp4",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (card tag; no LICENSE file in the repo); base google/gemma-4-26B-A4B-it Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.7",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "PixilabAI/Blink-v0.3-26B-A4B-NVFP4@08caad20c1574fb76341ca809ff38fdb081f7d4a",
   "noul_decisive": {
    "acc_among_decisive": 0.9467213114754098,
    "decisive": 244,
    "decisive_rate": 0.6703296703296703,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.67498392881862,
     "cost": 49.62845783533821,
     "intelligence": 49.21581563112197,
     "speed": 91.48151024194223
    },
    "board_release": "v1.6.43",
    "capability": 69.9453997799703,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 28,
    "capability_rank": 48,
    "composite_ci95": {
     "A": [
      45.23103654293889,
      64.78227929319183
     ],
     "B": [
      41.83419561295226,
      62.07842151813806
     ],
     "C": [
      31.410442043707555,
      52.48195783546016
     ]
    },
    "jevbench_score": 61.16059214787521,
    "measured_in": "v1.6.7",
    "method": "v1.6",
    "open_board_rank": 33,
    "rank": 57,
    "ranks": {
     "A": 57,
     "B": 57,
     "C": 62,
     "capability": 80
    },
    "scores": {
     "A": 61.16059214787521,
     "B": 57.67804533002703,
     "C": 42.47263343602446
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.252750841062516,
     "p50_s_raw": 0.05137542053125799,
     "p95_s_adjusted": 0.2812913428759202,
     "p95_s_raw": 0.06564567143796012
    },
    "views": {
     "Balanced 33:33:33": 55.71250293531964,
     "Emphasis on Accuracy 60:20:20": 51.855982453841044,
     "Emphasis on Cost 20:20:60": 52.04705676153498,
     "Emphasis on Speed 20:60:20": 65.14523654261257,
     "Intelligence only": 46.97811680206471
    }
   },
   "rank": 62,
   "ranked": true,
   "ranks": {
    "A": 62,
    "B": 64,
    "C": 72,
    "capability": 94
   },
   "repo": "https://huggingface.co/PixilabAI/Blink-v0.3-26B-A4B-NVFP4",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 61.12013368419001,
    "B": 57.64925493457254,
    "C": 42.44453728068751
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25477972798980775,
    "p50_s_raw": 0.05238986399490386,
    "p95_s_adjusted": 0.30209081431385126,
    "p95_s_raw": 0.07604540715692563
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8620689655172413,
       "cc": 78.57219756866152,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 66.66666666666667,
        "score": 77.38325937265121
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 78.57219756866152
      },
      "calibration_gen": {
       "accuracy": 0.6607142857142857,
       "cc": 44.41501762938165,
       "cc_by_type": {
        "choice": 69.72049689440995,
        "noul": 15.384615384615374,
        "score": 48.139940609119634
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 44.41501762938165
      },
      "composite_gen": {
       "accuracy": 0.7755102040816326,
       "cc": 61.661977649287216,
       "cc_by_type": {
        "choice": 81.48148148148148,
        "noul": 28.57142857142858,
        "score": 74.93302289495159
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 61.661977649287216
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 98.74247115687324,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.22741347061967
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 98.74247115687324
      },
      "jaggedness": {
       "accuracy": 0.5243243243243243,
       "cc": 12.830160086648034,
       "cc_by_type": {
        "choice": 55.53496989346919,
        "noul": -58.97435897435898,
        "score": 41.92986934083389
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 12.830160086648034
      },
      "llm_router": {
       "accuracy": 0.7992424242424242,
       "cc": 72.61663013184426,
       "cc_by_type": {
        "choice": 65.62935438922435,
        "noul": 75.34246575342465,
        "score": 76.87807025288377
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 72.61663013184426
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4838709677419355,
       "cc": 21.95983819644401,
       "cc_by_type": {
        "choice": 35.38461538461538,
        "noul": 0.0,
        "score": 30.494899204716653
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 21.95983819644401
      },
      "multilingual": {
       "accuracy": 0.6984126984126984,
       "cc": 49.44236254308296,
       "cc_by_type": {
        "choice": 72.2346618898343,
        "noul": 26.436781609195403,
        "score": 49.655644130219166
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 49.44236254308296
      },
      "negation_pair_gen": {
       "accuracy": 0.7321428571428571,
       "cc": 43.01413459029695,
       "cc_by_type": {
        "choice": 90.47619047619048,
        "noul": -9.999999999999998,
        "score": 48.56621329470037
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 43.01413459029695
      },
      "option_order": {
       "accuracy": 0.9649122807017544,
       "cc": 95.5604883462819,
       "cc_by_type": {
        "choice": 95.5604883462819
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 95.5604883462819
      },
      "perturb_pair": {
       "accuracy": 0.9,
       "cc": 84.88006962790548,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 85.71428571428572,
        "score": 84.6533150436116
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 84.88006962790548
      },
      "usecase": {
       "accuracy": 0.6286919831223629,
       "cc": 38.36326470609512,
       "cc_by_type": {
        "choice": 67.6238196184236,
        "noul": 19.298245614035082,
        "score": 28.167728885826694
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 38.36326470609512
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5454545454545454,
       "cc": 4.937825248418551,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": 0.0,
        "score": -5.186524254744351
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 4.937825248418551
      },
      "cs": {
       "accuracy": 0.75,
       "cc": 53.954802259887,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 53.954802259887
      },
      "da": {
       "accuracy": 0.7916666666666666,
       "cc": 54.43977410930783,
       "cc_by_type": {
        "choice": 64.49704142011835,
        "noul": 19.999999999999996,
        "score": 78.82228090780514
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 54.43977410930783
      },
      "de": {
       "accuracy": 0.8,
       "cc": 70.03622189564423,
       "cc_by_type": {
        "choice": 83.70549185274592,
        "noul": 77.77777777777777,
        "score": 48.625396056408995
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 70.03622189564423
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": -4.511613763017206,
       "cc_by_type": {
        "choice": 51.80722891566265,
        "noul": -60.0,
        "score": -5.3420702047142665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.6979445933869526,
       "cc": 46.31123906153604,
       "cc_by_type": {
        "choice": 70.16264722378011,
        "noul": 19.548872180451138,
        "score": 49.222197780376874
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 46.31123906153604
      },
      "es": {
       "accuracy": 0.6785714285714286,
       "cc": 54.13125875630315,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": 63.63636363636365,
        "score": 51.12104899618216
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 54.13125875630315
      },
      "fi": {
       "accuracy": 0.3333333333333333,
       "cc": -8.920187793427232,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.7333333333333333,
       "cc": 39.085828039995185,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -50.0,
        "score": 67.25748411998555
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 39.085828039995185
      },
      "hi": {
       "accuracy": 0.5333333333333333,
       "cc": 38.75519442187261,
       "cc_by_type": {
        "choice": 50.31055900621117,
        "noul": 19.999999999999996,
        "score": 45.95502425940667
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 38.75519442187261
      },
      "id": {
       "accuracy": 0.6666666666666666,
       "cc": 33.87096774193548,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 33.87096774193548
      },
      "it": {
       "accuracy": 0.8823529411764706,
       "cc": 76.94126084433556,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 30.82378253300666
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 76.94126084433556
      },
      "ja": {
       "accuracy": 0.53125,
       "cc": 25.083725989475457,
       "cc_by_type": {
        "choice": 59.34959349593497,
        "noul": -14.28571428571429,
        "score": 30.187298758205692
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 25.083725989475457
      },
      "ko": {
       "accuracy": 0.7272727272727273,
       "cc": 74.31438742885162,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 22.943162286554863
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 74.31438742885162
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.95012548973524,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.85037646920576
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 99.95012548973524
      },
      "nl": {
       "accuracy": 0.6923076923076923,
       "cc": 21.532452114485192,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -68.73597698987774
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 21.532452114485192
      },
      "no": {
       "accuracy": 0.5714285714285714,
       "cc": 33.87978142076503,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 33.87978142076503
      },
      "pl": {
       "accuracy": 0.65,
       "cc": 44.580498742442956,
       "cc_by_type": {
        "choice": 92.12942719720158,
        "noul": 0.0,
        "score": 41.612069030127294
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 44.580498742442956
      },
      "pt": {
       "accuracy": 0.9444444444444444,
       "cc": 90.22683698066082,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 100.0,
        "score": 88.77096320328903
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 90.22683698066082
      },
      "sv": {
       "accuracy": 0.625,
       "cc": 36.356054299978204,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 53.51260734437906
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 36.356054299978204
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 60.31616533251732,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 100.0,
        "score": 7.264285471236143
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 60.31616533251732
      },
      "uk": {
       "accuracy": 0.6363636363636364,
       "cc": 10.754585109986635,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -100.0,
        "score": 75.74201619952514
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 10.754585109986635
      },
      "zh": {
       "accuracy": 0.875,
       "cc": 62.19801371486293,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": -13.405958855411203
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 62.19801371486293
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7693333333333333,
       "cc": 70.11573674209708,
       "cc_by_type": {
        "choice": 70.11573674209708
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 70.11573674209708
      },
      "noul": {
       "accuracy": 0.616,
       "cc": 23.2,
       "cc_by_type": {
        "noul": 23.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 23.2
      },
      "score": {
       "accuracy": 0.6293333333333333,
       "cc": 48.580321460438455,
       "cc_by_type": {
        "score": 48.580321460438455
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 48.580321460438455
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.75,
       "cc": 43.75694683730911,
       "cc_by_type": {
        "choice": 70.34596375617794,
        "noul": 19.999999999999996,
        "score": 40.924876755749395
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 43.75694683730911
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 30.4794447005702,
       "cc_by_type": {
        "choice": 70.54009819967267,
        "noul": 19.999999999999996,
        "score": 0.8982359020379249
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 30.4794447005702
      },
      "customer_support": {
       "accuracy": 0.65,
       "cc": 47.88048103401948,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 25.0,
        "score": 50.89950761818749
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 47.88048103401948
      },
      "demand_forecasting": {
       "accuracy": 0.19230769230769232,
       "cc": -40.46675998594265,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100.0,
        "score": -25.01473778915324
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 30.134531521647336,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -19.999999999999996,
        "score": 47.670675310283606
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 30.134531521647336
      },
      "feature_extraction": {
       "accuracy": 0.5862068965517241,
       "cc": 40.83135316808231,
       "cc_by_type": {
        "choice": 65.81196581196579,
        "noul": 19.999999999999996,
        "score": 36.68209369228115
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 40.83135316808231
      },
      "financial_crime": {
       "accuracy": 0.6153846153846154,
       "cc": 43.836506452390005,
       "cc_by_type": {
        "choice": 73.91304347826087,
        "noul": 25.0,
        "score": 32.59647587890912
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 43.836506452390005
      },
      "gaming": {
       "accuracy": 0.5517241379310345,
       "cc": 13.446554121042317,
       "cc_by_type": {
        "choice": 52.475247524752476,
        "noul": -14.28571428571429,
        "score": 2.150129124088762
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 13.446554121042317
      },
      "insurance_claims": {
       "accuracy": 0.5,
       "cc": 39.067485028605155,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": 19.999999999999996,
        "score": 76.56753445089484
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 39.067485028605155
      },
      "knowledge_graphs": {
       "accuracy": 0.5652173913043478,
       "cc": 30.3227658858092,
       "cc_by_type": {
        "choice": 54.198473282442755,
        "noul": 19.999999999999996,
        "score": 16.769824374984854
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 30.3227658858092
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 30.591986352709828,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 0.0,
        "score": 8.036121659755512
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 30.591986352709828
      },
      "legal_compliance": {
       "accuracy": 0.6153846153846154,
       "cc": 61.8756462123043,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 18.41382388281453
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 61.8756462123043
      },
      "llm_guardrails": {
       "accuracy": 0.7931034482758621,
       "cc": 60.58510134359349,
       "cc_by_type": {
        "choice": 81.65137614678899,
        "noul": 25.0,
        "score": 75.10392788399149
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 60.58510134359349
      },
      "model_routing": {
       "accuracy": 0.86,
       "cc": 72.32255475621157,
       "cc_by_type": {
        "choice": 88.74296435272043,
        "noul": 72.72727272727273,
        "score": 55.497427188641524
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 72.32255475621157
      },
      "moderation_trust_safety": {
       "accuracy": 0.875,
       "cc": 69.1293608581969,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 19.999999999999996,
        "score": 87.38808257459071
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 69.1293608581969
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 38.80908171961838,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 0.0,
        "score": 46.577998927699404
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 38.80908171961838
      },
      "risk_assessment": {
       "accuracy": 0.38461538461538464,
       "cc": -15.99553483012555,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": -100.0,
        "score": 8.61716909452901
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.5625,
       "cc": 27.277877295181455,
       "cc_by_type": {
        "choice": 60.1593625498008,
        "noul": -14.28571428571429,
        "score": 35.959983621457866
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 27.277877295181455
      },
      "search_retrieval": {
       "accuracy": 0.9090909090909091,
       "cc": 84.20501734874512,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 100.0,
        "score": 66.93486111544775
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 84.20501734874512
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.77397078635877,
      90.58056567199262
     ],
     "I": [
      43.35399336962467,
      54.02584879853301
     ],
     "capability": [
      65.11784736173853,
      71.70425623732149
     ],
     "composite": {
      "A": [
       45.20260615882145,
       64.73830080500183
      ],
      "B": [
       41.81467673642534,
       62.04610921006097
      ],
      "C": [
       31.390698721403783,
       52.4463580470654
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.67498392881862,
     "capability": 69.9453997799703,
     "intelligence": 49.21581563112197
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7693333333333333,
      "calibration": 90.35563030686254,
      "cc": 70.11573674209708,
      "cc_by_type": {
       "choice": 70.11573674209708
      },
      "cc_tiered": 71.21947685199589,
      "cc_tiered_open": 74.01883434266433,
      "cc_tiered_sealed": 68.42011936132744,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.11573674209708
     },
     "noul": {
      "accuracy": 0.616,
      "calibration": 90.17262756314747,
      "cc": 23.2,
      "cc_by_type": {
       "noul": 23.2
      },
      "cc_tiered": 24.522870023442106,
      "cc_tiered_open": 28.205659075224297,
      "cc_tiered_sealed": 20.840080971659916,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 23.2
     },
     "score": {
      "accuracy": 0.6293333333333333,
      "calibration": 91.49669391644588,
      "cc": 48.580321460438455,
      "cc_by_type": {
       "score": 48.580321460438455
      },
      "cc_tiered": 51.90510001792792,
      "cc_tiered_open": 57.66440539777861,
      "cc_tiered_sealed": 46.145794638077234,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.580321460438455
     }
    },
    "raw_on_A": {
     "calibration": 87.63270878627716,
     "capability": 67.620060964045,
     "intelligence": 47.60741314181283
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 55.66774722403986,
    "Emphasis on Accuracy 60:20:20": 51.83270978561536,
    "Emphasis on Cost 20:20:60": 52.023612309991556,
    "Emphasis on Speed 20:60:20": 65.03518511056224,
    "Intelligence only": 46.97811680206471
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 48.5204641422835,
    "cost": 69.27998573050196,
    "intelligence": 28.182424563225457,
    "speed": 91.65069544774573
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.27998611987381716,
      "ece_score": 44.00277602523657,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 44.00277602523657,
      "score_before_time_limit": 44.00277602523657,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.06593406593406592,
      "brier": 0.25247848434065967,
      "ece": 0.2512450549450551,
      "n": 364,
      "score": 49.750989010988974,
      "score_before_time_limit": 49.750989010988974,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.7146739130434783,
      "interval90_width": 0.3179347826086956,
      "n": 368,
      "nrps": 0.19245016957880431,
      "score": 51.80762739062497,
      "score_before_time_limit": 51.80762739062497,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.38569864130434806
     }
    },
    "score": 48.5204641422835
   },
   "capability": 38.35144435275448,
   "composite_ci95": {
    "A": [
     8.910915927068583,
     24.382694954866707
    ],
    "B": [
     7.469168295495286,
     21.838354164060085
    ],
    "C": [
     6.188136060464294,
     16.932427051990775
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact same-base market reference; self-hosted open weights with no per-token tariff of their own. pricing_v15.reference() RAISES on every spelling of google/gemma-4-E4B-it - the base is in none of the three frozen tables - so this row carries its own LABELLED estimate from the reference the dated 25 Sep 2026 snapshot itself contains for exactly this base and size, deepinfra:google/gemma-4-E4B-it at USD 0.02 per 1M input and USD 0.1 per 1M output. The already-published classone-gemma4-e2b row publishes at that same rate as a one-size-UP estimate, so for this row it is the exact size rather than an upward approximation. RELEASE-LANE LINE, recorded and not acted on: the base needs either a listing decision or an UNLISTED_BASE_MODELS entry in the frozen table; no rate was invented here and the frozen module was not modified. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 792,614 input and 0 output tokens over 1,500 decisions. Estimated, not charged.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010568186666666667
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Gemma 4 E4B (Gemma-4-E4B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 33.31401280028791,
    "I_sealed": 23.050836326163,
    "base": 28.182424563225457,
    "excess": 0.1899318927851983,
    "gap": 10.263176474124911,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 35.64521709768209,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 20.729789241899976,
       "judge": 36.71720500988794,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 41.01932367149759,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 23.277497631684074,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 38.74636363636365,
       "hard": 25.742037037037047,
       "judge": 16.479661016949144,
       "standard": 20.81074074074073
      }
     },
     "sealed|choice": {
      "cc": 37.362851267778666,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 38.054020231515274,
       "hard": 33.71441134439924,
       "judge": 36.076302572167826,
       "standard": 46.243969676085456
      }
     },
     "sealed|noul": {
      "cc": 31.78965771071034,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 48.71794871794872,
       "hard": 10.606060606060597,
       "judge": 41.66666666666667,
       "standard": 50.877192982456144
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 14.18728971962614,
       "hard": -10.75497222222217,
       "judge": 1.1792035398229705,
       "standard": 4.336081081081044
      }
     }
    }
   },
   "jevbench_score": 15.60446295131971,
   "key": "classone-gemma4-e4b",
   "last_measured_on": "2026-10-10",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-gemma4-e4b@74d6e08152878a4956ce9ffc323f6c06f13ca9af",
   "noul_decisive": {
    "acc_among_decisive": 0.7382352941176471,
    "decisive": 340,
    "decisive_rate": 0.9340659340659341,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 48.5204641422835,
     "cost": 69.27998573050196,
     "intelligence": 28.182424563225457,
     "speed": 91.7664962211246
    },
    "board_release": "v1.6.43",
    "capability": 38.35144435275448,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 115,
    "capability_rank": 143,
    "composite_ci95": {
     "A": [
      8.912270303019026,
      24.387188108020116
     ],
     "B": [
      7.469929402481821,
      21.841243683018
     ],
     "C": [
      6.189076599318768,
      16.935547297236194
     ]
    },
    "jevbench_score": 15.607101626553092,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 120,
    "rank": 158,
    "ranks": {
     "A": 158,
     "B": 157,
     "C": 159,
     "capability": 207
    },
    "scores": {
     "A": 15.607101626553092,
     "B": 13.587651382976851,
     "C": 10.838265018439648
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.22830128003843128,
     "p50_s_raw": 0.03915064001921564,
     "p95_s_adjusted": 0.29163660332560537,
     "p95_s_raw": 0.07081830166280269
    },
    "views": {
     "Balanced 33:33:33": 15.672226976099346,
     "Emphasis on Accuracy 60:20:20": 12.054100187818078,
     "Emphasis on Cost 20:20:60": 17.712389550075887,
     "Emphasis on Speed 20:60:20": 19.22912598635703,
     "Intelligence only": 8.95354562245959
    }
   },
   "rank": 175,
   "ranked": true,
   "ranks": {
    "A": 175,
    "B": 174,
    "C": 176,
    "capability": 230
   },
   "repo": "https://huggingface.co/devops-thiago/classone-gemma4-e4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.60446295131971,
    "B": 13.58605130005878,
    "C": 10.836432605083132
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2288790670922026,
    "p50_s_raw": 0.0394395335461013,
    "p95_s_adjusted": 0.29876132757635787,
    "p95_s_raw": 0.07438066378817894
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6551724137931034,
       "cc": 48.403611111111104,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 66.66666666666667,
        "score": 11.877499999999985
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 48.403611111111104
      },
      "calibration_gen": {
       "accuracy": 0.42857142857142855,
       "cc": 2.0163035515209473,
       "cc_by_type": {
        "choice": 32.453416149068325,
        "noul": 23.076923076923084,
        "score": -49.481428571428566
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 2.0163035515209473
      },
      "composite_gen": {
       "accuracy": 0.5102040816326531,
       "cc": 24.397270723104047,
       "cc_by_type": {
        "choice": 40.74074074074074,
        "noul": 21.42857142857142,
        "score": 11.02249999999999
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 24.397270723104047
      },
      "fairness": {
       "accuracy": 0.7777777777777778,
       "cc": 78.00062745098039,
       "cc_by_type": {
        "choice": 64.70588235294117,
        "noul": 100.0,
        "score": 69.296
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 78.00062745098039
      },
      "jaggedness": {
       "accuracy": 0.3837837837837838,
       "cc": 11.209656087483806,
       "cc_by_type": {
        "choice": 18.017600741083836,
        "noul": 23.076923076923084,
        "score": -7.465555555555503
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 11.209656087483806
      },
      "llm_router": {
       "accuracy": 0.6363636363636364,
       "cc": 55.196760619175414,
       "cc_by_type": {
        "choice": 47.05062703204832,
        "noul": 61.64383561643836,
        "score": 56.895819209039544
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 55.196760619175414
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5161290322580645,
       "cc": 23.0149386845039,
       "cc_by_type": {
        "choice": 16.923076923076923,
        "noul": 50.0,
        "score": 2.1217391304347744
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 23.0149386845039
      },
      "multilingual": {
       "accuracy": 0.5365079365079365,
       "cc": 29.473782295572374,
       "cc_by_type": {
        "choice": 43.573667711598766,
        "noul": 40.22988505747127,
        "score": 4.617794117647089
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 29.473782295572374
      },
      "negation_pair_gen": {
       "accuracy": 0.48214285714285715,
       "cc": -16.955793650793655,
       "cc_by_type": {
        "choice": 19.047619047619047,
        "noul": 30.000000000000004,
        "score": -99.91500000000002
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.38596491228070173,
       "cc": 22.30854605993341,
       "cc_by_type": {
        "choice": 22.30854605993341
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 22.30854605993341
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 67.99352945343772,
       "cc_by_type": {
        "choice": 64.61336828309307,
        "noul": 71.42857142857142,
        "score": 67.93864864864865
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 67.99352945343772
      },
      "usecase": {
       "accuracy": 0.46835443037974683,
       "cc": 14.865840833856993,
       "cc_by_type": {
        "choice": 42.18539217575643,
        "noul": 12.280701754385959,
        "score": -9.868571428571403
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 14.865840833856993
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6363636363636364,
       "cc": 26.925555555555565,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": 50.0,
        "score": -15.88999999999996
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 26.925555555555565
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 7.90960451977401,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 7.90960451977401
      },
      "da": {
       "accuracy": 0.4166666666666667,
       "cc": 24.536203155818544,
       "cc_by_type": {
        "choice": 17.159763313609464,
        "noul": 60.00000000000001,
        "score": -3.5511538461538317
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 24.536203155818544
      },
      "de": {
       "accuracy": 0.5333333333333333,
       "cc": 21.35741575994569,
       "cc_by_type": {
        "choice": 56.54797827398914,
        "noul": 11.111111111111116,
        "score": -3.5868421052631883
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 21.35741575994569
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": 0.07861445783131475,
       "cc_by_type": {
        "choice": 27.710843373493972,
        "noul": -19.999999999999996,
        "score": -7.475000000000032
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.07861445783131475
      },
      "en": {
       "accuracy": 0.49508489722966936,
       "cc": 22.359010078249664,
       "cc_by_type": {
        "choice": 35.3897924845765,
        "noul": 28.57142857142858,
        "score": 3.1158091787439113
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 22.359010078249664
      },
      "es": {
       "accuracy": 0.7857142857142857,
       "cc": 58.303939393939395,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 100.0,
        "score": 1.0936363636363633
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 58.303939393939395
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 47.183098591549296,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": 100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 47.183098591549296
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 30.50427698300759,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 0.0,
        "score": 42.28857142857142
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 30.50427698300759
      },
      "hi": {
       "accuracy": 0.5,
       "cc": 33.68955033643892,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": 60.00000000000001,
        "score": 15.602812499999995
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 33.68955033643892
      },
      "id": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 17.741935483870968
      },
      "it": {
       "accuracy": 0.7058823529411765,
       "cc": 57.02479166666668,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 50.0,
        "score": 21.07437500000001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 57.02479166666668
      },
      "ja": {
       "accuracy": 0.375,
       "cc": -0.5040061217576456,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -42.85714285714286,
        "score": 22.645937500000002
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.7272727272727273,
       "cc": 77.51933333333334,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 32.55800000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 77.51933333333334
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 98.75666666666667,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.27000000000004
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 98.75666666666667
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": -15.442971216341691,
       "cc_by_type": {
        "choice": 24.79108635097493,
        "noul": 0.0,
        "score": -71.12
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 34.42622950819672,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 34.42622950819672
      },
      "pl": {
       "accuracy": 0.625,
       "cc": 50.14468360745798,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 80.0,
        "score": -5.954230769230784
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 50.14468360745798
      },
      "pt": {
       "accuracy": 0.6111111111111112,
       "cc": 49.75193662758235,
       "cc_by_type": {
        "choice": 45.7286432160804,
        "noul": 66.66666666666667,
        "score": 36.86049999999999
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 49.75193662758235
      },
      "sv": {
       "accuracy": 0.5,
       "cc": -7.282962962962962,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50.0,
        "score": -82.96
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 23.552514619883052,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 100.0,
        "score": -103.02666666666664
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 23.552514619883052
      },
      "uk": {
       "accuracy": 0.2727272727272727,
       "cc": -34.453968784838345,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": -16.405384615384612
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.625,
       "cc": 36.97171171171172,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": 100.0,
        "score": -24.219999999999974
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 36.97171171171172
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5173333333333333,
       "cc": 37.46761098635343,
       "cc_by_type": {
        "choice": 37.46761098635343
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 37.46761098635343
      },
      "noul": {
       "accuracy": 0.6693333333333333,
       "cc": 33.86666666666667,
       "cc_by_type": {
        "noul": 33.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 33.86666666666667
      },
      "score": {
       "accuracy": 0.3413333333333333,
       "cc": 3.9127145359018956,
       "cc_by_type": {
        "score": 3.9127145359018956
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 3.9127145359018956
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": -21.09269082921473,
       "cc_by_type": {
        "choice": 40.691927512355846,
        "noul": 19.999999999999996,
        "score": -123.97000000000004
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": 6.366578355155487,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": 60.00000000000001,
        "score": -32.880624999999995
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 6.366578355155487
      },
      "customer_support": {
       "accuracy": 0.6,
       "cc": 34.63327543424317,
       "cc_by_type": {
        "choice": 51.61290322580645,
        "noul": 25.0,
        "score": 27.286923076923063
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 34.63327543424317
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -1.8498271875380354,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": 0.0,
        "score": -9.163939393939401
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -8.013566650740566,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -60.0,
        "score": 10.493461538461535
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.5172413793103449,
       "cc": 16.519821289821277,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": 19.999999999999996,
        "score": -19.158484848484882
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 16.519821289821277
      },
      "financial_crime": {
       "accuracy": 0.5384615384615384,
       "cc": 24.41154782608697,
       "cc_by_type": {
        "choice": 73.91304347826087,
        "noul": 25.0,
        "score": -25.678399999999968
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 24.41154782608697
      },
      "gaming": {
       "accuracy": 0.3793103448275862,
       "cc": 6.763275341819888,
       "cc_by_type": {
        "choice": 16.831683168316832,
        "noul": 42.85714285714286,
        "score": -39.39900000000003
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 6.763275341819888
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": -46.41502645502644,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -60.0,
        "score": -99.87999999999995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.21739130434782608,
       "cc": -6.9660931689176,
       "cc_by_type": {
        "choice": -9.923664122137408,
        "noul": 19.999999999999996,
        "score": -30.97461538461539
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 7.34117073170732,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 0.0,
        "score": -29.195999999999998
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 7.34117073170732
      },
      "legal_compliance": {
       "accuracy": 0.5384615384615384,
       "cc": 5.378939247830284,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": 81.71058823529413
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 5.378939247830284
      },
      "llm_guardrails": {
       "accuracy": 0.6551724137931034,
       "cc": 45.51440602211244,
       "cc_by_type": {
        "choice": 72.4770642201835,
        "noul": 50.0,
        "score": 14.066153846153838
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 45.51440602211244
      },
      "model_routing": {
       "accuracy": 0.7466666666666667,
       "cc": 66.53999293560527,
       "cc_by_type": {
        "choice": 67.83704100777273,
        "noul": 77.27272727272727,
        "score": 54.510210526315795
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 66.53999293560527
      },
      "moderation_trust_safety": {
       "accuracy": 0.5625,
       "cc": 31.290701754385953,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 60.00000000000001,
        "score": -2.970000000000028
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 31.290701754385953
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 37.53701370098354,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 33.33333333333333,
        "score": 9.42846153846154
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 37.53701370098354
      },
      "risk_assessment": {
       "accuracy": 0.6153846153846154,
       "cc": 37.185966981132076,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 100.0,
        "score": -50.70625
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 37.185966981132076
      },
      "scientific_discovery": {
       "accuracy": 0.28125,
       "cc": 2.9136300828432273,
       "cc_by_type": {
        "choice": -3.585657370517929,
        "noul": 14.28571428571428,
        "score": -1.9591666666666674
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 2.9136300828432273
      },
      "search_retrieval": {
       "accuracy": 0.6818181818181818,
       "cc": 46.84529194088859,
       "cc_by_type": {
        "choice": 57.04057279236276,
        "noul": 66.66666666666667,
        "score": 16.828636363636363
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 46.84529194088859
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      42.95152007629359,
      53.0806033344027
     ],
     "I": [
      22.568457707555886,
      33.716071635910566
     ],
     "capability": [
      33.46902235931629,
      42.57940480198994
     ],
     "composite": {
      "A": [
       8.910915927068583,
       24.382694954866707
      ],
      "B": [
       7.469168295495286,
       21.838354164060085
      ],
      "C": [
       6.188136060464294,
       16.932427051990775
      ]
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 48.5204641422835,
     "capability": 38.35144435275448,
     "intelligence": 28.182424563225457
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5173333333333333,
      "calibration": 44.00277602523657,
      "cc": 37.46761098635343,
      "cc_by_type": {
       "choice": 37.46761098635343
      },
      "cc_tiered": 36.50403418273038,
      "cc_tiered_open": 35.64521709768209,
      "cc_tiered_sealed": 37.362851267778666,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.46761098635343
     },
     "noul": {
      "accuracy": 0.6693333333333333,
      "calibration": 49.750989010988974,
      "cc": 33.86666666666667,
      "cc_by_type": {
       "noul": 33.86666666666667
      },
      "cc_tiered": 36.404490691103966,
      "cc_tiered_open": 41.01932367149759,
      "cc_tiered_sealed": 31.78965771071034,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.86666666666667
     },
     "score": {
      "accuracy": 0.3413333333333333,
      "calibration": 51.80762739062497,
      "cc": 3.9127145359018956,
      "cc_by_type": {
       "score": 3.9127145359018956
      },
      "cc_tiered": 11.638748815842037,
      "cc_tiered_open": 23.277497631684074,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 3.9127145359018956
     }
    },
    "raw_on_A": {
     "calibration": 43.42824044580669,
     "capability": 35.61702293170774,
     "intelligence": 27.805805417608788
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.668679522665748,
    "Emphasis on Accuracy 60:20:20": 12.052840888224397,
    "Emphasis on Cost 20:20:60": 17.70967065478269,
    "Emphasis on Speed 20:60:20": 19.219515882376403,
    "Intelligence only": 8.95354562245959
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 32.62324971930184,
    "cost": 70.24966415777286,
    "intelligence": 5.814527660927306,
    "speed": 92.94182096467813
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.42988044164037864,
      "ece_score": 14.023911671924271,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 14.023911671924271,
      "score_before_time_limit": 14.023911671924271,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.12637362637362637,
      "brier": 0.353035391263736,
      "ece": 0.33436318681318694,
      "n": 364,
      "score": 33.12736263736261,
      "score_before_time_limit": 33.12736263736261,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.7309782608695652,
      "interval90_width": 0.45244565217391297,
      "n": 368,
      "nrps": 0.215767459549366,
      "score": 50.718474848618655,
      "score_before_time_limit": 50.718474848618655,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.3849315217391305
     }
    },
    "score": 32.62324971930184
   },
   "capability": 19.21888869011457,
   "composite_ci95": {
    "A": [
     0.0742086944842331,
     1.554043780168782
    ],
    "B": [
     0.051003926003689956,
     1.2101702767395142
    ],
    "C": [
     0.05153381561405077,
     1.0791970695616542
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-2B straight through to deepinfra:Qwen/Qwen3.5-2B at USD 0.02 per 1M input and USD 0.1 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 735,765 input and 0 output tokens over 1,500 decisions. Estimated, not charged.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009810200000000002
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Qwen 3.5 2B (Qwen3.5-2B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.892803202604126,
    "I_sealed": 4.736252119250485,
    "base": 5.814527660927306,
    "excess": -7.916693497986072,
    "gap": 2.1565510833536408,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 18.56541234955058,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.297297297297305,
       "hard": 16.955017301038072,
       "judge": 5.075807514831907,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 1.9710144927536213,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": -21.739130434782606,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 0.14198276550817734,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.27727272727272,
       "hard": -2.0812037037037046,
       "judge": 4.7081355932203195,
       "standard": -27.328518518518543
      }
     },
     "sealed|choice": {
      "cc": 10.956701395170182,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 19.282511210762333,
       "hard": 9.970319885676586,
       "judge": 3.2012581807112888,
       "standard": 20.399724328049636
      }
     },
     "sealed|noul": {
      "cc": 3.2520549625812754,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 38.46153846153846,
       "hard": -12.121212121212121,
       "judge": 8.333333333333325,
       "standard": 8.771929824561408
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 16.134112149532708,
       "hard": -13.1237222222222,
       "judge": -24.52305309734517,
       "standard": -9.89180180180178
      }
     }
    }
   },
   "jevbench_score": 0.2376395234121874,
   "key": "classone-qwen3.5-2b",
   "last_measured_on": "2026-10-10",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-qwen3.5-2b@fc15401b6ae6d86b8fdf664b40bfdd8055edf374",
   "noul_decisive": {
    "acc_among_decisive": 0.6226415094339622,
    "decisive": 318,
    "decisive_rate": 0.8736263736263736,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 32.62324971930184,
     "cost": 70.24966415777286,
     "intelligence": 5.814527660927306,
     "speed": 93.11279379161611
    },
    "board_release": "v1.6.43",
    "capability": 19.21888869011457,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 134,
    "capability_rank": 164,
    "composite_ci95": {
     "A": [
      0.07421330468385162,
      1.5542613561325538
     ],
     "B": [
      0.051005668190310965,
      1.2102758221999665
     ],
     "C": [
      0.051537017141563626,
      1.0793481639809404
     ]
    },
    "jevbench_score": 0.23766015025378578,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 168,
    "rank": 212,
    "ranks": {
     "A": 212,
     "B": 212,
     "C": 212,
     "capability": 236
    },
    "scores": {
     "A": 0.23766015025378578,
     "B": 0.1692155731407913,
     "C": 0.16504177100957346
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21041937693953514,
     "p50_s_raw": 0.03020968846976757,
     "p95_s_adjusted": 0.23207849023165178,
     "p95_s_raw": 0.0410392451158259
    },
    "views": {
     "Balanced 33:33:33": 0.2059858256814215,
     "Emphasis on Accuracy 60:20:20": 0.1250037108819809,
     "Emphasis on Cost 20:20:60": 0.29995214102056916,
     "Emphasis on Speed 20:60:20": 0.3095514049527884,
     "Intelligence only": 0.0786327227711625
    }
   },
   "rank": 234,
   "ranked": true,
   "ranks": {
    "A": 234,
    "B": 234,
    "C": 234,
    "capability": 258
   },
   "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.2376395234121874,
    "B": 0.16920720735254766,
    "C": 0.165027446814019
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2125162780052051,
    "p50_s_raw": 0.03125813900260255,
    "p95_s_adjusted": 0.23901530014118177,
    "p95_s_raw": 0.0445076500705909
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.4827586206896552,
       "cc": 26.97833333333334,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": 66.66666666666667,
        "score": 5.935000000000013
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 26.97833333333334
      },
      "calibration_gen": {
       "accuracy": 0.3392857142857143,
       "cc": -13.726849020544668,
       "cc_by_type": {
        "choice": 16.14906832298137,
        "noul": -15.384615384615385,
        "score": -41.944999999999986
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.32653061224489793,
       "cc": 0.7510267857142816,
       "cc_by_type": {
        "choice": 0.0,
        "noul": -7.14285714285714,
        "score": 9.395937499999985
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0.7510267857142816
      },
      "fairness": {
       "accuracy": 0.4444444444444444,
       "cc": -7.00355555555556,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -21.010666666666687
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0.0
      },
      "jaggedness": {
       "accuracy": 0.3837837837837838,
       "cc": 5.861599107426817,
       "cc_by_type": {
        "choice": 18.017600741083836,
        "noul": 7.692307692307687,
        "score": -8.125111111111071
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 5.861599107426817
      },
      "llm_router": {
       "accuracy": 0.48863636363636365,
       "cc": 33.92782003547969,
       "cc_by_type": {
        "choice": 21.96934509986067,
        "noul": 42.46575342465753,
        "score": 37.34836158192088
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 33.92782003547969
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3225806451612903,
       "cc": -20.34245261984394,
       "cc_by_type": {
        "choice": 16.923076923076923,
        "noul": -25.0,
        "score": -52.950434782608745
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.38412698412698415,
       "cc": 4.092899567611803,
       "cc_by_type": {
        "choice": 13.121361397223472,
        "noul": 8.045977011494255,
        "score": -8.888639705882317
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 4.092899567611803
      },
      "negation_pair_gen": {
       "accuracy": 0.39285714285714285,
       "cc": -26.347896825396816,
       "cc_by_type": {
        "choice": 9.523809523809527,
        "noul": 10.000000000000009,
        "score": -98.56749999999998
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.21052631578947367,
       "cc": 0.11098779134295962,
       "cc_by_type": {
        "choice": 0.11098779134295962
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 0.11098779134295962
      },
      "perturb_pair": {
       "accuracy": 0.5666666666666667,
       "cc": 48.77980199071942,
       "cc_by_type": {
        "choice": 25.29488859764089,
        "noul": 71.42857142857142,
        "score": 49.61594594594595
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 48.77980199071942
      },
      "usecase": {
       "accuracy": 0.3206751054852321,
       "cc": -11.511251481717963,
       "cc_by_type": {
        "choice": 6.340335324725405,
        "noul": -19.298245614035093,
        "score": -21.575844155844194
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -20.115,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": 0.0,
        "score": -80.345
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 49.152542372881356,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": 100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 49.152542372881356
      },
      "da": {
       "accuracy": 0.375,
       "cc": -13.013510848126225,
       "cc_by_type": {
        "choice": 28.994082840236683,
        "noul": -60.0,
        "score": -8.034615384615362
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.28888888888888886,
       "cc": -16.378460738952608,
       "cc_by_type": {
        "choice": 2.232951116475559,
        "noul": -33.333333333333336,
        "score": -18.035000000000046
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0.0
      },
      "el": {
       "accuracy": 0.2222222222222222,
       "cc": -34.461847389558244,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -60.0,
        "score": -47.00000000000002
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.3708668453976765,
       "cc": 2.919018796085924,
       "cc_by_type": {
        "choice": 12.955692652832235,
        "noul": 2.2556390977443552,
        "score": -6.454275362318818
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 2.919018796085924
      },
      "es": {
       "accuracy": 0.42857142857142855,
       "cc": -0.37590909090909125,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": -27.27272727272727,
        "score": -21.491363636363637
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 47.183098591549296,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": 100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 47.183098591549296
      },
      "fr": {
       "accuracy": 0.4,
       "cc": -2.847552779456862,
       "cc_by_type": {
        "choice": 23.83638928067701,
        "noul": 0.0,
        "score": -32.3790476190476
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.4,
       "cc": 11.143196169772247,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": 39.99999999999999,
        "score": -32.03625000000001
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 11.143196169772247
      },
      "id": {
       "accuracy": 0.3333333333333333,
       "cc": 35.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 35.483870967741936
      },
      "it": {
       "accuracy": 0.5882352941176471,
       "cc": 39.10954484015044,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 50.0,
        "score": 18.104374999999983
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 39.10954484015044
      },
      "ja": {
       "accuracy": 0.28125,
       "cc": -2.907849641889275,
       "cc_by_type": {
        "choice": -13.821138211382111,
        "noul": 14.28571428571428,
        "score": -9.188124999999992
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.36363636363636365,
       "cc": 15.881704918032781,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 0.0,
        "score": -19.56800000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 15.881704918032781
      },
      "mixed": {
       "accuracy": 0.7272727272727273,
       "cc": 58.55131868131869,
       "cc_by_type": {
        "choice": 56.04395604395604,
        "noul": 50.0,
        "score": 69.61000000000004
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 58.55131868131869
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": 5.939593933766626,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 33.33333333333333,
        "score": -15.236000000000027
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 5.939593933766626
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.475,
       "cc": 18.48718910116263,
       "cc_by_type": {
        "choice": 44.90599038041102,
        "noul": 39.99999999999999,
        "score": -29.444423076923123
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 18.48718910116263
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 18.251714405360136,
       "cc_by_type": {
        "choice": 45.7286432160804,
        "noul": 0.0,
        "score": 9.026500000000004
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 18.251714405360136
      },
      "sv": {
       "accuracy": 0.25,
       "cc": -12.537777777777777,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 0.0,
        "score": -4.279999999999995
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 0.6930994152046814,
       "cc_by_type": {
        "choice": 21.052631578947373,
        "noul": -33.333333333333336,
        "score": 14.360000000000007
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.6930994152046814
      },
      "uk": {
       "accuracy": 0.45454545454545453,
       "cc": -18.83422519509476,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": 30.453846153846154
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 47.33675675675678,
       "cc_by_type": {
        "choice": -29.729729729729726,
        "noul": 100.0,
        "score": 71.74000000000005
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 47.33675675675678
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.336,
       "cc": 13.974779754707205,
       "cc_by_type": {
        "choice": 13.974779754707205
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.974779754707205
      },
      "noul": {
       "accuracy": 0.528,
       "cc": 5.600000000000005,
       "cc_by_type": {
        "noul": 5.600000000000005
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 5.600000000000005
      },
      "score": {
       "accuracy": 0.32266666666666666,
       "cc": -8.548239929947377,
       "cc_by_type": {
        "score": -8.548239929947377
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2916666666666667,
       "cc": -53.64625846604429,
       "cc_by_type": {
        "choice": 11.037891268533771,
        "noul": 19.999999999999996,
        "score": -191.97666666666663
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -28.865504978177853,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": 19.999999999999996,
        "score": -98.576875
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.35,
       "cc": 2.0286683209263834,
       "cc_by_type": {
        "choice": -12.90322580645161,
        "noul": 25.0,
        "score": -6.010769230769242
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 2.0286683209263834
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": -8.96346355117439,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -50.0,
        "score": 19.495151515151534
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 24.71274167861124,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -19.999999999999996,
        "score": 50.03884615384615
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 24.71274167861124
      },
      "feature_extraction": {
       "accuracy": 0.27586206896551724,
       "cc": -17.316809116809114,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -19.999999999999996,
        "score": -37.93333333333333
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.46153846153846156,
       "cc": 3.7182956521739166,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": -25.0,
        "score": -11.671199999999992
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 3.7182956521739166
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -26.63343815810153,
       "cc_by_type": {
        "choice": 4.950495049504946,
        "noul": -42.85714285714286,
        "score": -41.993666666666684
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.14285714285714285,
       "cc": -7.78137566137566,
       "cc_by_type": {
        "choice": -26.984126984126984,
        "noul": -60.0,
        "score": 63.64000000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.08695652173913043,
       "cc": -29.252194167155995,
       "cc_by_type": {
        "choice": -28.244274809160306,
        "noul": -19.999999999999996,
        "score": -39.512307692307694
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": 6.393062330623304,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": 0.0,
        "score": 0.47999999999999154
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 6.393062330623304
      },
      "legal_compliance": {
       "accuracy": 0.07692307692307693,
       "cc": -39.32074895531984,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100.0,
        "score": 13.185294117647029
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.4827586206896552,
       "cc": 11.789371912491177,
       "cc_by_type": {
        "choice": 17.431192660550458,
        "noul": 25.0,
        "score": -7.0630769230769275
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 11.789371912491177
      },
      "model_routing": {
       "accuracy": 0.5466666666666666,
       "cc": 35.63095556818022,
       "cc_by_type": {
        "choice": 35.674082015545444,
        "noul": 63.63636363636365,
        "score": 7.582421052631572
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 35.63095556818022
      },
      "moderation_trust_safety": {
       "accuracy": 0.25,
       "cc": -50.34215538847118,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100.0,
        "score": -87.86857142857143
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6190476190476191,
       "cc": 44.91556070953055,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 66.66666666666667,
        "score": -1.7692307692307674
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 44.91556070953055
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": -22.14570230607967,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": -33.333333333333336,
        "score": -76.50000000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.25,
       "cc": 0.1523784228166729,
       "cc_by_type": {
        "choice": -19.52191235059761,
        "noul": 14.28571428571428,
        "score": 5.69333333333335
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.1523784228166729
      },
      "search_retrieval": {
       "accuracy": 0.36363636363636365,
       "cc": -15.631133289939976,
       "cc_by_type": {
        "choice": 14.08114558472553,
        "noul": 0.0,
        "score": -60.974545454545456
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      26.883939686902206,
      37.1769715121807
     ],
     "I": [
      3.840627882382685,
      11.691929733973469
     ],
     "capability": [
      16.023273303858527,
      23.5837139621214
     ],
     "composite": {
      "A": [
       0.0742086944842331,
       1.554043780168782
      ],
      "B": [
       0.051003926003689956,
       1.2101702767395142
      ],
      "C": [
       0.05153381561405077,
       1.0791970695616542
      ]
     },
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 32.62324971930184,
     "capability": 19.21888869011457,
     "intelligence": 5.814527660927306
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 14.023911671924271,
      "cc": 13.974779754707205,
      "cc_by_type": {
       "choice": 13.974779754707205
      },
      "cc_tiered": 14.761056872360381,
      "cc_tiered_open": 18.56541234955058,
      "cc_tiered_sealed": 10.956701395170182,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.974779754707205
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 33.12736263736261,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 2.6115347276674483,
      "cc_tiered_open": 1.9710144927536213,
      "cc_tiered_sealed": 3.2520549625812754,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.32266666666666666,
      "calibration": 50.718474848618655,
      "cc": -8.548239929947377,
      "cc_by_type": {
       "score": -8.548239929947377
      },
      "cc_tiered": 0.07099138275408867,
      "cc_tiered_open": 0.14198276550817734,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 30.618919596314058,
     "capability": 18.126890524291774,
     "intelligence": 5.634861452269493
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.20596516581341576,
    "Emphasis on Accuracy 60:20:20": 0.12499914549463981,
    "Emphasis on Cost 20:20:60": 0.2999258556833412,
    "Emphasis on Speed 20:60:20": 0.30946743639927554,
    "Intelligence only": 0.0786327227711625
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 48.84618651494331,
    "cost": 64.96692638610241,
    "intelligence": 27.338167706853277,
    "speed": 91.03995916839054
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.28518548895899043,
      "ece_score": 42.96290220820191,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 42.96290220820191,
      "score_before_time_limit": 42.96290220820191,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.14010989010989006,
      "brier": 0.27496240302197805,
      "ece": 0.25560109890109894,
      "n": 364,
      "score": 48.87978021978021,
      "score_before_time_limit": 48.87978021978021,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.720108695652174,
      "interval90_width": 0.30865036231884047,
      "n": 368,
      "nrps": 0.18476832722826092,
      "score": 54.69587711684781,
      "score_before_time_limit": 54.69587711684781,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.3606570652173914
     }
    },
    "score": 48.84618651494331
   },
   "capability": 38.09217711089829,
   "composite_ci95": {
    "A": [
     7.88683793216811,
     20.321076853319873
    ],
    "B": [
     6.6169077395655185,
     17.99108358885262
    ],
    "C": [
     5.476970786227854,
     14.111858925916582
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line.; reference `deepinfra:Qwen/Qwen3.5-4B` frozen at the 25 Sep cut-off, deprecated afterwards (replaced by `Qwen/Qwen3.5-9B`); re-score if it changes (I-3) OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 735,765 input and 0 output tokens over 1,500 decisions. Estimated, not charged.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014715299999999999
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Qwen 3.5 4B (Qwen3.5-4B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 36.145599118617284,
    "I_sealed": 18.530736295089266,
    "base": 27.338167706853277,
    "excess": 7.541618242188305,
    "gap": 17.614862823528018,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 59.400926298363274,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 64.97297297297298,
       "hard": 41.49103491664046,
       "judge": 64.40342781806197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 24.24913733609386,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 13.043478260869556,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 24.786733721394732,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.40090909090911,
       "hard": 29.234259259259243,
       "judge": 20.891525423728808,
       "standard": 14.4274074074074
      }
     },
     "sealed|choice": {
      "cc": 38.6857940545428,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 45.56262383981645,
       "hard": 36.68242277673958,
       "judge": 33.33671553954645,
       "standard": 47.27773949000691
      }
     },
     "sealed|noul": {
      "cc": 16.457796589375537,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 38.46153846153846,
       "hard": -4.545454545454541,
       "judge": 30.555555555555557,
       "standard": 26.315789473684205
      }
     },
     "sealed|score": {
      "cc": 0.4486182413494726,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 28.812056074766346,
       "hard": -10.255555555555485,
       "judge": -2.3791150442477926,
       "standard": 11.916846846846852
      }
     }
    }
   },
   "jevbench_score": 14.33333238845815,
   "key": "classone-qwen3.5-4b",
   "last_measured_on": "2026-10-10",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-qwen3.5-4b@6626fdea7a3e8786bad8d2d5cc8acded1756a17c",
   "noul_decisive": {
    "acc_among_decisive": 0.7092651757188498,
    "decisive": 313,
    "decisive_rate": 0.8598901098901099,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 48.84618651494331,
     "cost": 64.96692638610241,
     "intelligence": 27.338167706853277,
     "speed": 91.46618934008248
    },
    "board_release": "v1.6.43",
    "capability": 38.09217711089829,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 116,
    "capability_rank": 144,
    "composite_ci95": {
     "A": [
      7.891109860006282,
      20.334524220090714
     ],
     "B": [
      6.6193128804538945,
      17.999509630335773
     ],
     "C": [
      5.479937402782141,
      14.121197375063
     ]
    },
    "jevbench_score": 14.342131800224653,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 122,
    "rank": 161,
    "ranks": {
     "A": 161,
     "B": 161,
     "C": 162,
     "capability": 208
    },
    "scores": {
     "A": 14.342131800224653,
     "B": 12.460860262475673,
     "C": 9.959813750156009
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23571948995813727,
     "p50_s_raw": 0.04285974497906864,
     "p95_s_adjusted": 0.3026812879485078,
     "p95_s_raw": 0.07634064397425391
    },
    "views": {
     "Balanced 33:33:33": 14.257369523966407,
     "Emphasis on Accuracy 60:20:20": 10.98579554132928,
     "Emphasis on Cost 20:20:60": 15.954348902584416,
     "Emphasis on Speed 20:60:20": 17.632944190866688,
     "Intelligence only": 8.172749758440709
    }
   },
   "rank": 180,
   "ranked": true,
   "ranks": {
    "A": 180,
    "B": 181,
    "C": 182,
    "capability": 231
   },
   "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 14.33333238845815,
    "B": 12.455545381775147,
    "C": 9.953703047540383
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2453273990890011,
    "p50_s_raw": 0.04766369954450056,
    "p95_s_adjusted": 0.3208174840314314,
    "p95_s_raw": 0.08540874201571569
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6206896551724138,
       "cc": 43.73972222222221,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 33.33333333333333,
        "score": 39.55249999999998
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 43.73972222222221
      },
      "calibration_gen": {
       "accuracy": 0.38392857142857145,
       "cc": -2.796751473164511,
       "cc_by_type": {
        "choice": 30.124223602484474,
        "noul": 7.692307692307687,
        "score": -46.206785714285694
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.5510204081632653,
       "cc": 31.21112378747796,
       "cc_by_type": {
        "choice": 48.14814814814815,
        "noul": 35.71428571428572,
        "score": 9.770937499999999
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 31.21112378747796
      },
      "fairness": {
       "accuracy": 0.7777777777777778,
       "cc": 37.86533333333333,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -66.66666666666667,
        "score": 80.26266666666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 37.86533333333333
      },
      "jaggedness": {
       "accuracy": 0.42702702702702705,
       "cc": 3.439985798532349,
       "cc_by_type": {
        "choice": 37.47105141269106,
        "noul": -38.46153846153846,
        "score": 11.310444444444446
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 3.439985798532349
      },
      "llm_router": {
       "accuracy": 0.6628787878787878,
       "cc": 56.02907910025096,
       "cc_by_type": {
        "choice": 55.411054342777526,
        "noul": 53.42465753424656,
        "score": 59.2515254237288
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 56.02907910025096
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5161290322580645,
       "cc": 15.55809364548495,
       "cc_by_type": {
        "choice": 26.153846153846153,
        "noul": 50.0,
        "score": -29.4795652173913
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 15.55809364548495
      },
      "multilingual": {
       "accuracy": 0.5015873015873016,
       "cc": 20.949561234629694,
       "cc_by_type": {
        "choice": 36.40841916703987,
        "noul": 28.735632183908045,
        "score": -2.2953676470588213
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 20.949561234629694
      },
      "negation_pair_gen": {
       "accuracy": 0.42857142857142855,
       "cc": -45.491190476190475,
       "cc_by_type": {
        "choice": 28.571428571428573,
        "noul": 0.0,
        "score": -165.04500000000002
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.6666666666666666,
       "cc": 57.82463928967813,
       "cc_by_type": {
        "choice": 57.82463928967813
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 57.82463928967813
      },
      "perturb_pair": {
       "accuracy": 0.8166666666666667,
       "cc": 78.77666619437262,
       "cc_by_type": {
        "choice": 72.47706422018348,
        "noul": 85.71428571428572,
        "score": 78.13864864864864
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 78.77666619437262
      },
      "usecase": {
       "accuracy": 0.459915611814346,
       "cc": 13.14458872834252,
       "cc_by_type": {
        "choice": 31.778762767392575,
        "noul": 8.771929824561408,
        "score": -1.1169264069264218
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 13.14458872834252
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6363636363636364,
       "cc": 13.13722222222224,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": 50.0,
        "score": -57.254999999999946
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 13.13722222222224
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 7.90960451977401,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 7.90960451977401
      },
      "da": {
       "accuracy": 0.5416666666666666,
       "cc": 36.62549309664694,
       "cc_by_type": {
        "choice": 40.8284023668639,
        "noul": 100.0,
        "score": -30.951923076923073
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 36.62549309664694
      },
      "de": {
       "accuracy": 0.5111111111111111,
       "cc": 22.671439359983808,
       "cc_by_type": {
        "choice": 40.25347012673505,
        "noul": 11.111111111111116,
        "score": 16.649736842105256
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 22.671439359983808
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": -25.492449799196788,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -19.999999999999996,
        "score": -72.14
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.5192135835567471,
       "cc": 21.206639325438474,
       "cc_by_type": {
        "choice": 43.690409422321885,
        "noul": 9.774436090225569,
        "score": 10.155072463767967
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 21.206639325438474
      },
      "es": {
       "accuracy": 0.7142857142857143,
       "cc": 55.06090909090909,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": 100.0,
        "score": 17.546363636363637
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 55.06090909090909
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 24.4131455399061,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 24.4131455399061
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 26.589757539122846,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 0.0,
        "score": 5.15714285714286
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 26.589757539122846
      },
      "hi": {
       "accuracy": 0.4,
       "cc": 9.455742753623186,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": 19.999999999999996,
        "score": -4.676250000000004
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 9.455742753623186
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -14.516129032258062,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 59.36320992007523,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50.0,
        "score": 53.4775
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 59.36320992007523
      },
      "ja": {
       "accuracy": 0.3125,
       "cc": 1.1244367015098795,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -14.28571428571429,
        "score": 15.220000000000022
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 1.1244367015098795
      },
      "ko": {
       "accuracy": 0.7272727272727273,
       "cc": 77.836,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 33.507999999999996
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 77.836
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 98.28333333333332,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 94.84999999999997
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 98.28333333333332
      },
      "nl": {
       "accuracy": 0.3076923076923077,
       "cc": -7.494183844011142,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 0.0,
        "score": -22.204
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5714285714285714,
       "cc": 33.87978142076503,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 33.87978142076503
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 30.731736262935428,
       "cc_by_type": {
        "choice": 68.51770878880629,
        "noul": 60.00000000000001,
        "score": -36.32250000000001
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 30.731736262935428
      },
      "pt": {
       "accuracy": 0.7222222222222222,
       "cc": 60.06590480178671,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 66.66666666666667,
        "score": 31.62150000000001
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 60.06590480178671
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 0.943703703703702,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50.0,
        "score": -58.28
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.943703703703702
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 3.5850292397660724,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": 33.33333333333333,
        "score": -69.94666666666669
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 3.5850292397660724
      },
      "uk": {
       "accuracy": 0.09090909090909091,
       "cc": -59.95774804905239,
       "cc_by_type": {
        "choice": -30.43478260869565,
        "noul": -100.0,
        "score": -49.43846153846154
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 1.0,
       "cc": 69.91333333333333,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 9.739999999999993
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 69.91333333333333
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5573333333333333,
       "cc": 42.6498531698048,
       "cc_by_type": {
        "choice": 42.6498531698048
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 42.6498531698048
      },
      "noul": {
       "accuracy": 0.592,
       "cc": 18.399999999999995,
       "cc_by_type": {
        "noul": 18.399999999999995
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.399999999999995
      },
      "score": {
       "accuracy": 0.384,
       "cc": 6.636094570928175,
       "cc_by_type": {
        "score": 6.636094570928175
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 6.636094570928175
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4166666666666667,
       "cc": -33.318480688266526,
       "cc_by_type": {
        "choice": 11.037891268533771,
        "noul": 19.999999999999996,
        "score": -130.99333333333334
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": 0.8938591107473973,
       "cc_by_type": {
        "choice": 1.8003273322422266,
        "noul": 60.00000000000001,
        "score": -59.11875000000004
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.8938591107473973
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 25.121935483870967,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 25.0,
        "score": 47.13999999999999
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 25.121935483870967
      },
      "demand_forecasting": {
       "accuracy": 0.34615384615384615,
       "cc": 2.214011196300356,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": 0.0,
        "score": 3.0275757575757734
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 2.214011196300356
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4444444444444444,
       "cc": 10.85620321707278,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -19.999999999999996,
        "score": 8.469230769230762
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 10.85620321707278
      },
      "feature_extraction": {
       "accuracy": 0.27586206896551724,
       "cc": -14.564156954156951,
       "cc_by_type": {
        "choice": 23.076923076923077,
        "noul": -60.0,
        "score": -6.769393939393931
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.38461538461538464,
       "cc": 6.2751362318840584,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0.0,
        "score": -15.957200000000004
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 6.2751362318840584
      },
      "gaming": {
       "accuracy": 0.4482758620689655,
       "cc": 11.478893603646071,
       "cc_by_type": {
        "choice": 28.712871287128706,
        "noul": 42.85714285714286,
        "score": -37.133333333333354
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 11.478893603646071
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": -53.55068783068785,
       "cc_by_type": {
        "choice": 36.507936507936506,
        "noul": -100.0,
        "score": -97.16000000000005
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.43478260869565216,
       "cc": 7.427903699354076,
       "cc_by_type": {
        "choice": 26.717557251908392,
        "noul": 19.999999999999996,
        "score": -24.43384615384616
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 7.427903699354076
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 20.725170731707326,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 25.0,
        "score": -14.043999999999969
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 20.725170731707326
      },
      "legal_compliance": {
       "accuracy": 0.15384615384615385,
       "cc": -29.474503375120538,
       "cc_by_type": {
        "choice": -14.754098360655735,
        "noul": -100.0,
        "score": 26.330588235294115
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5172413793103449,
       "cc": 22.654965890378737,
       "cc_by_type": {
        "choice": 44.95412844036697,
        "noul": 25.0,
        "score": -1.9892307692307654
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 22.654965890378737
      },
      "model_routing": {
       "accuracy": 0.7333333333333333,
       "cc": 62.21502323863556,
       "cc_by_type": {
        "choice": 67.83704100777273,
        "noul": 68.18181818181819,
        "score": 50.626210526315795
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 62.21502323863556
      },
      "moderation_trust_safety": {
       "accuracy": 0.6875,
       "cc": 29.053859649122796,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 100.0,
        "score": -65.47000000000003
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 29.053859649122796
      },
      "recruiting": {
       "accuracy": 0.7142857142857143,
       "cc": 37.555988059957905,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 33.33333333333333,
        "score": 9.485384615384618
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 37.555988059957905
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": -1.4943134171907815,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -81.21250000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.40625,
       "cc": 17.67869126667931,
       "cc_by_type": {
        "choice": 12.350597609561751,
        "noul": 42.85714285714286,
        "score": -2.171666666666683
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 17.67869126667931
      },
      "search_retrieval": {
       "accuracy": 0.5909090909090909,
       "cc": 36.62560810973699,
       "cc_by_type": {
        "choice": 42.72076372315036,
        "noul": 33.33333333333333,
        "score": 33.82272727272727
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 36.62560810973699
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      43.368446520267796,
      53.193644571845724
     ],
     "I": [
      21.59802304927416,
      31.44548663789592
     ],
     "capability": [
      33.4245046846555,
      41.60857704511476
     ],
     "composite": {
      "A": [
       7.88683793216811,
       20.321076853319873
      ],
      "B": [
       6.6169077395655185,
       17.99108358885262
      ],
      "C": [
       5.476970786227854,
       14.111858925916582
      ]
     },
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 48.84618651494331,
     "capability": 38.09217711089829,
     "intelligence": 27.338167706853277
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5573333333333333,
      "calibration": 42.96290220820191,
      "cc": 42.6498531698048,
      "cc_by_type": {
       "choice": 42.6498531698048
      },
      "cc_tiered": 49.043360176453035,
      "cc_tiered_open": 59.400926298363274,
      "cc_tiered_sealed": 38.6857940545428,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 42.6498531698048
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 48.87978021978021,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 20.3534669627347,
      "cc_tiered_open": 24.24913733609386,
      "cc_tiered_sealed": 16.457796589375537,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.384,
      "calibration": 54.69587711684781,
      "cc": 6.636094570928175,
      "cc_by_type": {
       "score": 6.636094570928175
      },
      "cc_tiered": 12.617675981372102,
      "cc_tiered_open": 24.786733721394732,
      "cc_tiered_sealed": 0.4486182413494726,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 6.636094570928175
     }
    },
    "raw_on_A": {
     "calibration": 44.37411603270306,
     "capability": 35.50100696428197,
     "intelligence": 26.62789789586089
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.245777558981105,
    "Emphasis on Accuracy 60:20:20": 10.98166428234479,
    "Emphasis on Cost 20:20:60": 15.945637186564964,
    "Emphasis on Speed 20:60:20": 17.601060567410833,
    "Intelligence only": 8.172749758440709
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Cloudflare",
   "axes": {
    "calibration": 82.15469329918429,
    "cost": 26.792900075221908,
    "intelligence": 34.47716016566848,
    "speed": 86.57770472711044
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09821189024390234,
      "ece_score": 80.35762195121954,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 80.35762195121954,
      "score_before_time_limit": 80.35762195121954,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3493333333333334,
      "brier": 0.1572640221333334,
      "ece": 0.06652586666666661,
      "n": 375,
      "score": 86.69482666666669,
      "score_before_time_limit": 86.69482666666669,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8506666666666667,
      "interval90_width": 0.36177777777777775,
      "n": 375,
      "nrps": 0.11951350773999989,
      "score": 79.41163127966668,
      "score_before_time_limit": 79.41163127966668,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.14612693333333326
     }
    },
    "score": 82.15469329918429
   },
   "capability": 58.315926732426384,
   "composite_ci95": {
    "A": [
     3.881235660057879,
     8.583323218918355
    ],
    "B": [
     3.561448407121937,
     8.315991449234165
    ],
    "C": [
     2.69530254170686,
     5.96064112424886
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-class reference. Self-hosted open weights; the declared base Qwen/Qwen3-Omni-30B-A3B-Instruct is not a key of any frozen pricing table. Method as the v1.6.5 Liquid rows: the nearest same-family entry of the frozen 25 Sep 2026 snapshot, deepinfra:Qwen/Qwen3-VL-30B-A3B-Instruct (the multimodal Qwen3 30B-A3B), at USD 0.15 per 1M input and USD 0.60 per 1M output. The authors' systemone() has no generation path, so output_tokens is a literal 0. This row's own usage on draw v1.6-regular-20261010-a2: 2,755,730 input and 0 output tokens over 1,500 decisions.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.275573
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Clef-omni",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX PRO 6000 Blackwell 96 GB pod (Beltsville US, 29 min, USD 1.19/h); the authors' own load_release_model() + systemone() from the pinned repo's joint_schema_model.py behind the run-39 loopback shim (127.0.0.1, single-threaded, no answer arithmetic), pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, defaults unchanged (max_length 64,000). Stack as the model card tested: torch 2.11.0+cu130, transformers 5.10.2, plus torchvision 0.26.0+cu130, which the Omni video processor requires to load. HF/transformers offline, no credentials, pod-level iptables egress cut installed and verified before the sealed input was uploaded. All 17 LFS files verified against the Hugging Face tree digests. GPU memory 62,729 MiB after load and at most 74,735 MiB during the run (per-minute log), of 97,887 MiB.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 37.25444425506104,
    "I_sealed": 31.699876076275928,
    "base": 34.47716016566848,
    "excess": -4.518676402554602,
    "gap": 5.554568178785111,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 65.02240890008018,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243245,
       "hard": 67.9144385026738,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 10.814354727398204,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -13.043478260869568,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.92656913770473,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.82848484848485,
       "hard": 42.1251851851852,
       "judge": 49.07338983050846,
       "standard": -4.141851851851852
      }
     },
     "sealed|choice": {
      "cc": 53.547209374932514,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 62.45698195849411,
       "hard": 49.543805650214345,
       "judge": 52.513824767896104,
       "standard": 58.64920744314267
      }
     },
     "sealed|noul": {
      "cc": 9.67795362532205,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 43.58974358974359,
       "hard": -10.606060606060607,
       "judge": 16.666666666666675,
       "standard": 22.807017543859654
      }
     },
     "sealed|score": {
      "cc": 31.87446522857323,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 45.32728971962617,
       "hard": 22.845027777777847,
       "judge": 36.40668141592918,
       "standard": 36.408603603603595
      }
     }
    }
   },
   "jevbench_score": 6.064517244374852,
   "key": "clef-omni",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.6",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Cloudflare/clef-omni@0db1cd2607d76a7bdb2a382f659e7b313079f84b",
   "noul_decisive": {
    "acc_among_decisive": 0.8565573770491803,
    "decisive": 244,
    "decisive_rate": 0.6506666666666666,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.15469329918429,
     "cost": 26.792900075221908,
     "intelligence": 34.47716016566848,
     "speed": 86.7880661703893
    },
    "board_release": "v1.6.43",
    "capability": 58.315926732426384,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      3.8823605492743414,
      8.586118919617187
     ],
     "B": [
      3.562206076177268,
      8.318090708030555
     ],
     "C": [
      2.696083714773848,
      5.9625825830674914
     ]
    },
    "jevbench_score": 6.06640325953113,
    "measured_in": "v1.6.6",
    "method": "v1.6",
    "open_board_rank": 143,
    "rank": 183,
    "ranks": {
     "A": 183,
     "B": 182,
     "C": 184,
     "capability": 141
    },
    "scores": {
     "A": 6.06640325953113,
     "B": 5.735167687307061,
     "C": 4.2127800413410625
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.43618169901892545,
     "p50_s_raw": 0.14309084950946271,
     "p95_s_adjusted": 0.4803147715982049,
     "p95_s_raw": 0.16515738579910247
    },
    "views": {
     "Balanced 33:33:33": 5.261179060414329,
     "Emphasis on Accuracy 60:20:20": 5.02460048275502,
     "Emphasis on Cost 20:20:60": 4.476423118785528,
     "Emphasis on Speed 20:60:20": 6.765856602997514,
     "Intelligence only": 4.70710504931535
    }
   },
   "rank": 204,
   "ranked": true,
   "ranks": {
    "A": 204,
    "B": 204,
    "C": 206,
    "capability": 159
   },
   "repo": "https://huggingface.co/Cloudflare/clef-omni",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 6.064517244374852,
    "B": 5.733819041964475,
    "C": 4.211470308593648
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.44394626477733257,
    "p50_s_raw": 0.14697313238866627,
    "p95_s_adjusted": 0.49533511972986155,
    "p95_s_raw": 0.1726675598649308
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.896551724137931,
       "cc": 86.57097222222222,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 68.04625
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 86.57097222222222
      },
      "calibration_gen": {
       "accuracy": 0.5982142857142857,
       "cc": 32.91760471412644,
       "cc_by_type": {
        "choice": 55.745341614906835,
        "noul": 15.384615384615374,
        "score": 27.622857142857114
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 32.91760471412644
      },
      "composite_gen": {
       "accuracy": 0.6632653061224489,
       "cc": 50.659661596119925,
       "cc_by_type": {
        "choice": 74.07407407407408,
        "noul": 35.71428571428572,
        "score": 42.19062499999998
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 50.659661596119925
      },
      "fairness": {
       "accuracy": 0.7407407407407407,
       "cc": 51.41265359477125,
       "cc_by_type": {
        "choice": 88.23529411764707,
        "noul": 0.0,
        "score": 66.00266666666667
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 51.41265359477125
      },
      "jaggedness": {
       "accuracy": 0.42162162162162165,
       "cc": -0.8300322403138493,
       "cc_by_type": {
        "choice": 30.523390458545624,
        "noul": -53.84615384615385,
        "score": 20.832666666666675
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6818181818181818,
       "cc": 57.872019428772546,
       "cc_by_type": {
        "choice": 50.76637250348352,
        "noul": 53.42465753424656,
        "score": 69.42502824858757
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 57.872019428772546
      },
      "long_state": {
       "accuracy": 0.3103448275862069,
       "cc": -1.3167821067821077,
       "cc_by_type": {
        "choice": 16.66666666666667,
        "noul": -27.27272727272727,
        "score": 6.655714285714276
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5483870967741935,
       "cc": 5.659007803790407,
       "cc_by_type": {
        "choice": 53.84615384615384,
        "noul": -25.0,
        "score": -11.869130434782615
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 5.659007803790407
      },
      "multilingual": {
       "accuracy": 0.5714285714285714,
       "cc": 29.241308106053285,
       "cc_by_type": {
        "choice": 54.32154052843709,
        "noul": 5.747126436781613,
        "score": 27.655257352941153
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 29.241308106053285
      },
      "negation_pair_gen": {
       "accuracy": 0.6607142857142857,
       "cc": 18.345555555555563,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 19.999999999999996,
        "score": -31.62999999999998
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 18.345555555555563
      },
      "option_order": {
       "accuracy": 0.8947368421052632,
       "cc": 86.68146503884572,
       "cc_by_type": {
        "choice": 86.68146503884572
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 86.68146503884572
      },
      "perturb_pair": {
       "accuracy": 0.9,
       "cc": 81.10219498659866,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 71.42857142857142,
        "score": 87.6054054054054
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 81.10219498659866
      },
      "usecase": {
       "accuracy": 0.5232067510548524,
       "cc": 17.11602027853846,
       "cc_by_type": {
        "choice": 53.74831374060514,
        "noul": -15.789473684210531,
        "score": 13.389220779220778
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 17.11602027853846
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -23.164444444444445,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -50.0,
        "score": -66.16
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 7.90960451977401,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 7.90960451977401
      },
      "da": {
       "accuracy": 0.5416666666666666,
       "cc": 23.599881656804737,
       "cc_by_type": {
        "choice": 52.662721893491124,
        "noul": -19.999999999999996,
        "score": 38.13692307692308
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 23.599881656804737
      },
      "de": {
       "accuracy": 0.7111111111111111,
       "cc": 51.80671574665221,
       "cc_by_type": {
        "choice": 72.84248642124321,
        "noul": 55.55555555555556,
        "score": 27.02210526315789
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 51.80671574665221
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": -55.7336546184739,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -100.0,
        "score": -106.96
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.6094727435210009,
       "cc": 34.60248155699925,
       "cc_by_type": {
        "choice": 56.70218732473357,
        "noul": 9.774436090225569,
        "score": 37.33082125603862
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 34.60248155699925
      },
      "es": {
       "accuracy": 0.7142857142857143,
       "cc": 54.17545454545455,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 63.63636363636365,
        "score": 25.071818181818184
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 54.17545454545455
      },
      "fi": {
       "accuracy": 0.1111111111111111,
       "cc": -52.816901408450704,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": -100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 17.07420198356729,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -50.0,
        "score": 26.610476190476174
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 17.07420198356729
      },
      "hi": {
       "accuracy": 0.6333333333333333,
       "cc": 44.937223084886135,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 60.00000000000001,
        "score": 12.078749999999994
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 44.937223084886135
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -48.387096774193544,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7058823529411765,
       "cc": 50.644668253408554,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50.0,
        "score": 27.321874999999995
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 50.644668253408554
      },
      "ja": {
       "accuracy": 0.46875,
       "cc": 18.159518002322883,
       "cc_by_type": {
        "choice": 26.829268292682926,
        "noul": -14.28571428571429,
        "score": 41.93500000000001
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 18.159518002322883
      },
      "ko": {
       "accuracy": 0.6363636363636364,
       "cc": 66.1930382513661,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 31.365999999999982
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 66.1930382513661
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 85.82333333333332,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 57.469999999999956
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 85.82333333333332
      },
      "nl": {
       "accuracy": 0.5384615384615384,
       "cc": 15.964908077994421,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0.0,
        "score": -1.9660000000000233
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 15.964908077994421
      },
      "no": {
       "accuracy": 0.2857142857142857,
       "cc": -15.846994535519126,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 36.12173351608311,
       "cc_by_type": {
        "choice": 84.25885439440316,
        "noul": 19.999999999999996,
        "score": 4.106346153846174
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 36.12173351608311
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 74.19423813512005,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 66.66666666666667,
        "score": 74.0065
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 74.19423813512005
      },
      "sv": {
       "accuracy": 0.375,
       "cc": 4.158888888888879,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 50.0,
        "score": -4.190000000000027
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 4.158888888888879
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 22.35695906432748,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -39.94666666666669
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 22.35695906432748
      },
      "uk": {
       "accuracy": 0.7272727272727273,
       "cc": 12.17157190635451,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": -100.0,
        "score": 58.25384615384615
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 12.17157190635451
      },
      "zh": {
       "accuracy": 0.625,
       "cc": 50.52282282282281,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": 33.33333333333333,
        "score": 83.09999999999995
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 50.52282282282281
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6653333333333333,
       "cc": 56.64190706512352,
       "cc_by_type": {
        "choice": 56.64190706512352
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 56.64190706512352
      },
      "noul": {
       "accuracy": 0.5573333333333333,
       "cc": 11.466666666666669,
       "cc_by_type": {
        "noul": 11.466666666666669
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 11.466666666666669
      },
      "score": {
       "accuracy": 0.5333333333333333,
       "cc": 33.64992119089322,
       "cc_by_type": {
        "score": 33.64992119089322
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 33.64992119089322
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5416666666666666,
       "cc": 8.511753615229727,
       "cc_by_type": {
        "choice": 40.691927512355846,
        "noul": -60.0,
        "score": 44.843333333333334
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 8.511753615229727
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 9.050032733224224,
       "cc_by_type": {
        "choice": 70.54009819967267,
        "noul": -19.999999999999996,
        "score": -23.39
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 9.050032733224224
      },
      "customer_support": {
       "accuracy": 0.3,
       "cc": -13.059735318444998,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -75.0,
        "score": 0.336923076923068
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": -8.969614214433495,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -50.0,
        "score": -16.667878787878788
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5555555555555556,
       "cc": 47.39004937091894,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 60.00000000000001,
        "score": 38.07076923076923
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 47.39004937091894
      },
      "feature_extraction": {
       "accuracy": 0.4827586206896552,
       "cc": 12.539619269619266,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": -60.0,
        "score": 48.90090909090908
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 12.539619269619266
      },
      "financial_crime": {
       "accuracy": 0.5384615384615384,
       "cc": 14.042347826086965,
       "cc_by_type": {
        "choice": 73.91304347826087,
        "noul": 0.0,
        "score": -31.78599999999998
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 14.042347826086965
      },
      "gaming": {
       "accuracy": 0.41379310344827586,
       "cc": 5.618274556027025,
       "cc_by_type": {
        "choice": 28.712871287128706,
        "noul": -14.28571428571429,
        "score": 2.4276666666666613
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 5.618274556027025
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": -11.208359788359795,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -100.0,
        "score": 45.73999999999998
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.5217391304347826,
       "cc": 9.780414954002737,
       "cc_by_type": {
        "choice": 45.03816793893129,
        "noul": -60.0,
        "score": 44.30307692307692
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 9.780414954002737
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 5.368834688346889,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -75.0,
        "score": 7.3666666666666885
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 5.368834688346889
      },
      "legal_compliance": {
       "accuracy": 0.3076923076923077,
       "cc": -20.515570556091294,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": 4.027058823529394
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.6206896551724138,
       "cc": 32.95168666196189,
       "cc_by_type": {
        "choice": 63.30275229357799,
        "noul": 0.0,
        "score": 35.55230769230767
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 32.95168666196189
      },
      "model_routing": {
       "accuracy": 0.74,
       "cc": 63.43091944512483,
       "cc_by_type": {
        "choice": 69.44518895738408,
        "noul": 77.27272727272727,
        "score": 43.57484210526314
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 63.43091944512483
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 24.84468671679197,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": -19.999999999999996,
        "score": 73.48142857142854
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 24.84468671679197
      },
      "recruiting": {
       "accuracy": 0.6190476190476191,
       "cc": 25.69128720525705,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 0.0,
        "score": 7.2246153846153955
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 25.69128720525705
      },
      "risk_assessment": {
       "accuracy": 0.5384615384615384,
       "cc": 14.524019916142551,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -33.15750000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 14.524019916142551
      },
      "scientific_discovery": {
       "accuracy": 0.3125,
       "cc": -9.035004163241226,
       "cc_by_type": {
        "choice": 28.286852589641434,
        "noul": -42.85714285714286,
        "score": -12.53472222222225
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.8181818181818182,
       "cc": 63.02598587305031,
       "cc_by_type": {
        "choice": 71.36038186157518,
        "noul": 66.66666666666667,
        "score": 51.05090909090908
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 63.02598587305031
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.99498334146031,
      83.8879680508817
     ],
     "I": [
      28.570607529410697,
      40.0809890185093
     ],
     "capability": [
      53.56177237020907,
      61.07325750582405
     ],
     "composite": {
      "A": [
       3.881235660057879,
       8.583323218918355
      ],
      "B": [
       3.561448407121937,
       8.315991449234165
      ],
      "C": [
       2.69530254170686,
       5.96064112424886
      ]
     },
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.15469329918429,
     "capability": 58.315926732426384,
     "intelligence": 34.47716016566848
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6653333333333333,
      "calibration": 80.35762195121954,
      "cc": 56.64190706512352,
      "cc_by_type": {
       "choice": 56.64190706512352
      },
      "cc_tiered": 59.28480913750634,
      "cc_tiered_open": 65.02240890008018,
      "cc_tiered_sealed": 53.547209374932514,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.64190706512352
     },
     "noul": {
      "accuracy": 0.5573333333333333,
      "calibration": 86.69482666666669,
      "cc": 11.466666666666669,
      "cc_by_type": {
       "noul": 11.466666666666669
      },
      "cc_tiered": 10.246154176360127,
      "cc_tiered_open": 10.814354727398204,
      "cc_tiered_sealed": 9.67795362532205,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 11.466666666666669
     },
     "score": {
      "accuracy": 0.5333333333333333,
      "calibration": 79.41163127966668,
      "cc": 33.64992119089322,
      "cc_by_type": {
       "score": 33.64992119089322
      },
      "cc_tiered": 33.90051718313898,
      "cc_tiered_open": 35.92656913770473,
      "cc_tiered_sealed": 31.87446522857323,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.64992119089322
     }
    },
    "raw_on_A": {
     "calibration": 80.93316891446167,
     "capability": 59.00548965209646,
     "intelligence": 37.07781038973123
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 5.259287734304745,
    "Emphasis on Accuracy 60:20:20": 5.02356528996005,
    "Emphasis on Cost 20:20:60": 4.475601462482863,
    "Emphasis on Speed 20:60:20": 6.76022913396346,
    "Intelligence only": 4.70710504931535
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Omar / @kouhxp (hi@kouh.me)",
   "axes": {
    "calibration": 75.89600269702092,
    "cost": 78.3542548810428,
    "intelligence": 8.600978834309654,
    "speed": 71.56789504904872
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08639168454258676,
      "ece_score": 82.72166309148264,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 81.50841203280756,
      "score_before_time_limit": 82.72166309148264,
      "time_limit_failed_share": 0.014666666666666666
     },
     "noul": {
      "abstention_rate": 0.706043956043956,
      "brier": 0.24853405620583507,
      "ece": 0.12309816483516484,
      "n": 364,
      "score": 73.1692096,
      "score_before_time_limit": 75.38036703296703,
      "time_limit_failed_share": 0.029333333333333333
     },
     "score": {
      "interval90_coverage": 0.8695652173913043,
      "interval90_width": 0.6397192028985506,
      "n": 368,
      "nrps": 0.16768646620403746,
      "score": 73.01038645825521,
      "score_before_time_limit": 74.39917098327638,
      "time_limit_failed_share": 0.018666666666666668,
      "top_ece": 0.17216505706521745
     }
    },
    "score": 75.89600269702092
   },
   "capability": 42.24849076566529,
   "composite_ci95": {
    "A": [
     0.35621566725965836,
     1.8669624572598496
    ],
    "B": [
     0.24857514103500242,
     1.390185737453484
    ],
    "C": [
     0.2473719911525405,
     1.2965017064304514
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), input tokens only. The author's gutsy-inference 0.3.0 server emits no output-token field in any spelling; under the chief-of-staff decision of 9 Oct 2026 a missing output_tokens field on a native option-probability model that generates no text is treated as 0 output tokens, exactly as a server reporting a literal 0 is treated. Zero output is a verified property of this backend, not an assumption: it has no generation path at all (no generate/create_completion/n_predict/max_tokens/sampler); it prefills via llm.eval(tokens) and reads llama_get_logits_ith(ctx,-1) (r52 PREREG-AMENDMENT-4). Rate: the declared base Qwen/Qwen3.5-0.8B resolves in the frozen 25 Sep 2026 snapshot to deepinfra:Qwen/Qwen3.5-0.8B at USD 0.01 per 1M input and USD 0.05 per 1M output. This row's own measured usage on draw v1.6-regular-20261010-a2: 789,980 input tokens and 0 output over 1,500 decisions => USD 0.0052665333333333335 per 1,000 decisions.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0052665333333333335
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 29,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Gutsy 0.8B v0.3",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium pod (2xRTX3090 node, Germany), CPU-only by the author's own config (n_gpu_layers 0), 32 vCPU, egress cut with iptables and proven before the sealed input was uploaded; the author's own gutsy-inference 0.3.0 with their models.example.json settings unchanged (n_ctx 8192, reject_slot true, n_threads null)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 9.070301583187167,
    "I_sealed": 8.131656085432141,
    "base": 8.600978834309654,
    "excess": -9.134599083584687,
    "gap": 0.9386454977550258,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 25.412389147780978,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 37.72972972972974,
       "hard": 20.729789241899976,
       "judge": 12.986156888595916,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -47.82608695652174,
       "judge": -60.0,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 1.7985156017805268,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.240515151515154,
       "hard": -7.74469814814811,
       "judge": 1.9498728813559296,
       "standard": -2.5630925925926196
      }
     },
     "sealed|choice": {
      "cc": 24.394968256296423,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 23.036813014912926,
       "hard": 20.853028470924478,
       "judge": 26.031150119222776,
       "standard": 29.70365265334253
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -43.58974358974359,
       "hard": -78.78787878787878,
       "judge": -47.22222222222222,
       "standard": -61.40350877192983
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 12.014685981308393,
       "hard": -2.484919999999935,
       "judge": -12.305056637168143,
       "standard": 12.255661711711696
      }
     }
    }
   },
   "jevbench_score": 0.7578766201128447,
   "key": "gutsy-0.8b-v0.3",
   "last_measured_on": "2026-10-10",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "kouhxp/gutsy@d325eb1742cc1c200e04b24417f192a0dbf74f91",
   "noul_decisive": {
    "acc_among_decisive": 0.7383177570093458,
    "decisive": 107,
    "decisive_rate": 0.29395604395604397,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.50040036924202,
     "cost": 78.3542548810428,
     "intelligence": 8.600978834309654,
     "speed": 72.8877498600171
    },
    "board_release": "v1.6.43",
    "capability": 43.05068960177584,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.35718114749129054,
      1.8750127120910802
     ],
     "B": [
      0.2489508080898962,
      1.3935931889582263
     ],
     "C": [
      0.24804246353561837,
      1.3020921611743612
     ]
    },
    "jevbench_score": 0.7604367004496922,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 158,
    "rank": 201,
    "ranks": {
     "A": 201,
     "B": 201,
     "C": 201,
     "capability": 196
    },
    "scores": {
     "A": 0.7604367004496922,
     "B": 0.5441131491342173,
     "C": 0.5280810419789529
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.6613200119999418,
     "p50_s_raw": 0.755660005999971,
     "p95_s_adjusted": 3.0957915995999423,
     "p95_s_raw": 1.4728957997999712
    },
    "views": {
     "Balanced 33:33:33": 0.6218800588368827,
     "Emphasis on Accuracy 60:20:20": 0.39424898336691744,
     "Emphasis on Cost 20:20:60": 0.8792467571253542,
     "Emphasis on Speed 20:60:20": 0.8693565458528719,
     "Intelligence only": 0.25450928339078893
    }
   },
   "rank": 223,
   "ranked": true,
   "ranks": {
    "A": 223,
    "B": 223,
    "C": 223,
    "capability": 224
   },
   "repo": "https://huggingface.co/kouhxp/gutsy",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.7578766201128447,
    "B": 0.543063071290183,
    "C": 0.5263032084116976
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.901271966999866,
    "p50_s_raw": 0.875635983499933,
    "p95_s_adjusted": 3.6657787574997194,
    "p95_s_raw": 1.7578893787498597
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5862068965517241,
       "cc": 30.238494444444445,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 0.0,
        "score": 32.38215000000001
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 30.238494444444445
      },
      "calibration_gen": {
       "accuracy": 0.38392857142857145,
       "cc": -19.57648651457238,
       "cc_by_type": {
        "choice": 41.770186335403736,
        "noul": -92.3076923076923,
        "score": -8.191953571428567
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.2653061224489796,
       "cc": -24.789154706790118,
       "cc_by_type": {
        "choice": 29.62962962962963,
        "noul": -100.0,
        "score": -3.9970937499999915
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.5555555555555556,
       "cc": 21.924596601307183,
       "cc_by_type": {
        "choice": 52.94117647058823,
        "noul": 0.0,
        "score": 12.83261333333332
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 21.924596601307183
      },
      "jaggedness": {
       "accuracy": 0.2756756756756757,
       "cc": -23.766898037764125,
       "cc_by_type": {
        "choice": 9.680407596109312,
        "noul": -79.48717948717949,
        "score": -1.4939222222221993
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.4583333333333333,
       "cc": 21.153469908954868,
       "cc_by_type": {
        "choice": 26.61402693915468,
        "noul": 1.3698630136986356,
        "score": 35.47651977401129
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 21.153469908954868
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.25806451612903225,
       "cc": -38.646075919732446,
       "cc_by_type": {
        "choice": 26.153846153846153,
        "noul": -100.0,
        "score": -42.09207391304348
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.346031746031746,
       "cc": -10.63069831663855,
       "cc_by_type": {
        "choice": 30.138826690550832,
        "noul": -58.62068965517242,
        "score": -3.4102319852940655
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.26785714285714285,
       "cc": -26.649703571428574,
       "cc_by_type": {
        "choice": 14.285714285714286,
        "noul": -90.0,
        "score": -4.234825000000009
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.17543859649122806,
       "cc": -4.328523862375131,
       "cc_by_type": {
        "choice": -4.328523862375131
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 0.0
      },
      "perturb_pair": {
       "accuracy": 0.5166666666666667,
       "cc": 19.74922901184277,
       "cc_by_type": {
        "choice": 41.02228047182176,
        "noul": -28.57142857142857,
        "score": 46.79683513513512
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 19.74922901184277
      },
      "usecase": {
       "accuracy": 0.3206751054852321,
       "cc": -16.990773799131347,
       "cc_by_type": {
        "choice": 27.153594141453095,
        "noul": -75.43859649122807,
        "score": -2.6873190476190745
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.36363636363636365,
       "cc": -27.022149999999996,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -50.0,
        "score": -51.06644999999999
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": -12.711864406779661,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.20833333333333334,
       "cc": -32.610591814595644,
       "cc_by_type": {
        "choice": 5.325443786982243,
        "noul": -100.0,
        "score": -3.1572192307691926
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.4666666666666667,
       "cc": 2.2274148960461893,
       "cc_by_type": {
        "choice": 40.25347012673505,
        "noul": -33.333333333333336,
        "score": -0.23789210526314886
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 2.2274148960461893
      },
      "el": {
       "accuracy": 0.2222222222222222,
       "cc": -51.29191646586346,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100.0,
        "score": -69.53840000000002
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.34405719392314565,
       "cc": -11.632396312202536,
       "cc_by_type": {
        "choice": 22.602355580482268,
        "noul": -61.654135338345874,
        "score": 4.154590821255999
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.5,
       "cc": 13.68897878787879,
       "cc_by_type": {
        "choice": 60.72727272727273,
        "noul": -27.27272727272727,
        "score": 7.612390909090916
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 13.68897878787879
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.3333333333333333,
       "cc": -15.758361508496193,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -100.0,
        "score": -21.887214285714272
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.3,
       "cc": -16.665771829710142,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": -60.0,
        "score": -3.0407937499999926
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.6470588235294118,
       "cc": 27.99738442348378,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 0.0,
        "score": 34.76789375
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 27.99738442348378
      },
      "ja": {
       "accuracy": 0.3125,
       "cc": -23.373968770567163,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -71.42857142857143,
        "score": -17.39252187499998
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.2727272727272727,
       "cc": -31.130543497267748,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": -27.81785999999997
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.36363636363636365,
       "cc": -3.6486553113553115,
       "cc_by_type": {
        "choice": 34.06593406593406,
        "noul": -50.0,
        "score": 4.988100000000006
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "nl": {
       "accuracy": 0.46153846153846156,
       "cc": -0.1424452553389107,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0.0,
        "score": -50.288060000000016
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 0.5464480874316955,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0.5464480874316955
      },
      "pl": {
       "accuracy": 0.4,
       "cc": 12.299376097900057,
       "cc_by_type": {
        "choice": 60.64713598600787,
        "noul": -19.999999999999996,
        "score": -3.7490076923077043
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 12.299376097900057
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 26.514546072026793,
       "cc_by_type": {
        "choice": 45.7286432160804,
        "noul": 0.0,
        "score": 33.81499499999999
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 26.514546072026793
      },
      "sv": {
       "accuracy": 0.0,
       "cc": -56.24097777777778,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -35.3896
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 0.7387181286549614,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": 33.33333333333333,
        "score": -78.48560000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.7387181286549614
      },
      "uk": {
       "accuracy": 0.09090909090909091,
       "cc": -46.02425072463768,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100.0,
        "score": -29.3771
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.125,
       "cc": -50.39295435435435,
       "cc_by_type": {
        "choice": -29.729729729729726,
        "noul": -33.333333333333336,
        "score": -88.11580000000001
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.41733333333333333,
       "cc": 24.512005527724998,
       "cc_by_type": {
        "choice": 24.512005527724998
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 24.512005527724998
      },
      "noul": {
       "accuracy": 0.21066666666666667,
       "cc": -57.86666666666667,
       "cc_by_type": {
        "noul": -57.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.3466666666666667,
       "cc": 0.9913860770578653,
       "cc_by_type": {
        "score": 0.9913860770578653
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.9913860770578653
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3333333333333333,
       "cc": -20.792305967417175,
       "cc_by_type": {
        "choice": 30.807248764415156,
        "noul": -60.0,
        "score": -33.184166666666684
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.16666666666666666,
       "cc": -51.04649872817785,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": -60.0,
        "score": -85.11985625
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.4,
       "cc": 7.8360928866832085,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0,
        "score": 38.02440769230768
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 7.8360928866832085
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -39.86741871729342,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100.0,
        "score": -11.168521212121219
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -17.362241654722098,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100.0,
        "score": 3.8138961538461302
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.1724137931034483,
       "cc": -29.604258430458433,
       "cc_by_type": {
        "choice": -2.564102564102566,
        "noul": -60.0,
        "score": -26.248672727272737
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.38461538461538464,
       "cc": -8.077263014492752,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": -100.0,
        "score": 27.942124
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -33.71579672167217,
       "cc_by_type": {
        "choice": 16.831683168316832,
        "noul": -100.0,
        "score": -17.97907333333333
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.14285714285714285,
       "cc": -70.40589841269842,
       "cc_by_type": {
        "choice": 4.761904761904763,
        "noul": -100.0,
        "score": -115.9796
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.21739130434782608,
       "cc": -16.350229066353496,
       "cc_by_type": {
        "choice": -9.923664122137408,
        "noul": -60.0,
        "score": 20.872976923076926
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": -2.498773089430904,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -75.0,
        "score": -16.23615666666669
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.38461538461538464,
       "cc": -26.543207810993252,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": -14.055852941176482
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.3793103448275862,
       "cc": -5.703711032698185,
       "cc_by_type": {
        "choice": 44.95412844036697,
        "noul": -50.0,
        "score": -12.065261538461524
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.5,
       "cc": 20.346316406896474,
       "cc_by_type": {
        "choice": 42.106673813990895,
        "noul": 9.090909090909083,
        "score": 9.841366315789447
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 20.346316406896474
      },
      "moderation_trust_safety": {
       "accuracy": 0.3125,
       "cc": -21.78233634085213,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100.0,
        "score": -2.1891142857142887
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.38095238095238093,
       "cc": -7.3411204741657015,
       "cc_by_type": {
        "choice": 54.77386934673366,
        "noul": -100.0,
        "score": 23.20276923076924
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.5384615384615384,
       "cc": 28.47671559224318,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 33.33333333333333,
        "score": -10.167337500000006
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 28.47671559224318
      },
      "scientific_discovery": {
       "accuracy": 0.21875,
       "cc": -30.3334952412572,
       "cc_by_type": {
        "choice": 12.350597609561751,
        "noul": -100.0,
        "score": -3.35108333333336
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.18181818181818182,
       "cc": -40.61265147175815,
       "cc_by_type": {
        "choice": 14.08114558472553,
        "noul": -100.0,
        "score": -35.91909999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      69.67939224880561,
      78.1660305687077
     ],
     "I": [
      6.552688842251461,
      11.980624454535823
     ],
     "capability": [
      38.73036369799716,
      44.464298745938066
     ],
     "composite": {
      "A": [
       0.35621566725965836,
       1.8669624572598496
      ],
      "B": [
       0.24857514103500242,
       1.390185737453484
      ],
      "C": [
       0.2473719911525405,
       1.2965017064304514
      ]
     },
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.89600269702092,
     "capability": 42.24849076566529,
     "intelligence": 8.600978834309654
    },
    "per_type": {
     "choice": {
      "accuracy": 0.41733333333333333,
      "calibration": 81.50841203280756,
      "cc": 24.512005527724998,
      "cc_by_type": {
       "choice": 24.512005527724998
      },
      "cc_tiered": 24.9036787020387,
      "cc_tiered_open": 25.412389147780978,
      "cc_tiered_sealed": 24.394968256296423,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 24.512005527724998
     },
     "noul": {
      "accuracy": 0.21066666666666667,
      "calibration": 73.1692096,
      "cc": -57.86666666666667,
      "cc_by_type": {
       "noul": -57.86666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.3466666666666667,
      "calibration": 73.01038645825521,
      "cc": 0.9913860770578653,
      "cc_by_type": {
       "score": 0.9913860770578653
      },
      "cc_tiered": 0.8992578008902634,
      "cc_tiered_open": 1.7985156017805268,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.9913860770578653
     }
    },
    "raw_on_A": {
     "calibration": 77.32547272800478,
     "capability": 42.983248163593274,
     "intelligence": 8.641023599181757
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.6207797349760219,
    "Emphasis on Accuracy 60:20:20": 0.3939833540079143,
    "Emphasis on Cost 20:20:60": 0.8779266880849766,
    "Emphasis on Speed 20:60:20": 0.8654963260793376,
    "Intelligence only": 0.25450928339078893
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "eljiwo / jiwidi (fhjaime96@gmail.com)",
   "axes": {
    "calibration": 78.54210067220151,
    "cost": 78.40361168685378,
    "intelligence": 15.872104931583502,
    "speed": 94.31864739858605
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14318770225758568,
      "ece_score": 71.36245954848286,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 71.36245954848286,
      "score_before_time_limit": 71.36245954848286,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.6428571428571428,
      "brier": 0.22635226125046812,
      "ece": 0.11375593371020894,
      "n": 364,
      "score": 77.24881325795822,
      "score_before_time_limit": 77.24881325795822,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9646739130434783,
      "interval90_width": 0.731884057971014,
      "n": 368,
      "nrps": 0.13949242305496334,
      "score": 87.01502921016349,
      "score_before_time_limit": 87.01502921016349,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.060103496370883344
     }
    },
    "score": 78.54210067220151
   },
   "capability": 47.20710280189251,
   "composite_ci95": {
    "A": [
     2.0717818904784333,
     6.329209242448164
    ],
    "B": [
     1.5337127235876,
     4.973633390507989
    ],
    "C": [
     1.4387374239433561,
     4.3952841961445595
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-0.8B straight through to deepinfra:Qwen/Qwen3.5-0.8B at USD 0.01 per 1M input and USD 0.05 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 786,993 input and 0 output tokens over 1,500 decisions. Estimated, not charged.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0052466200000000004
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jiwo 0.8b (Qwen3.5-0.8B backbone with a trained decision head, one forward pass, native probabilities, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own install line from their model card (pip install 'jiwo[cuda] @ git+https://github.com/jiwidi/jiwo@8716abab') and their own `jiwo serve`, nothing patched. The server reads JIWO_CHECKPOINT and calls from_pretrained with NO revision and there is no JIWO_REVISION setting, so it was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY and the served revision cannot have drifted to main. JIWO_MAX_LENGTH=65536, the author's own documented value for their published runs: the 8,192 default refuses longer prompts with HTTP 400 and never truncates. The trained context is 8,192 tokens, so the longest items of this draw are answered outside the range the model saw in training. CUDA graphs are OFF (JIWO_CUDA_GRAPHS unset): the author's own card says graphs agree with eager on only 99.7 % of fields, so eager is the exact path and the author's 14.5 ms headline uses a mode this measurement does not use. Requests are serialised by the author's own asyncio.Lock. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 20.53318982032003,
    "I_sealed": 11.211020042846975,
    "base": 15.872104931583502,
    "excess": -0.7510748038666595,
    "gap": 9.322169777473054,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 35.509112308155075,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 41.62162162162163,
       "hard": 30.16671909405474,
       "judge": 32.76203032300594,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -39.13043478260869,
       "judge": -30.000000000000004,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 26.09045715280502,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.4799625425419,
       "hard": 18.20775252838498,
       "judge": 31.25102514541178,
       "standard": 21.920261717866516
      }
     },
     "sealed|choice": {
      "cc": 24.483755785527055,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 32.4225675252894,
       "hard": 18.37968561064086,
       "judge": 25.117954441682315,
       "standard": 31.771192281185392
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -17.948717948717952,
       "hard": -69.6969696969697,
       "judge": -33.333333333333336,
       "standard": -54.385964912280706
      }
     },
     "sealed|score": {
      "cc": 9.149304343013874,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 26.922451761233834,
       "hard": 2.576868911219099,
       "judge": 8.09382448748016,
       "standard": 14.99082128079402
      }
     }
    }
   },
   "jevbench_score": 4.067693241125661,
   "key": "jiwo-0.8b",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (weights, model card); MIT (code, github.com/jiwidi/jiwo)",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "eljiwo/jiwo-0.8b@52d217f8d68114e853b31ca364f8cb7da1b3c64d",
   "noul_decisive": {
    "acc_among_decisive": 0.7615384615384615,
    "decisive": 130,
    "decisive_rate": 0.35714285714285715,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.54210067220151,
     "cost": 78.40361168685378,
     "intelligence": 15.872104931583502,
     "speed": 94.33479301471377
    },
    "board_release": "v1.6.43",
    "capability": 47.20710280189251,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 98,
    "capability_rank": 125,
    "composite_ci95": {
     "A": [
      2.071813842220017,
      6.329337632277144
     ],
     "B": [
      1.533726731769256,
      4.973696434425022
     ],
     "C": [
      1.4387596126527897,
      4.395373355748017
     ]
    },
    "jevbench_score": 4.067767731552672,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 145,
    "rank": 185,
    "ranks": {
     "A": 185,
     "B": 187,
     "C": 186,
     "capability": 184
    },
    "scores": {
     "A": 4.067767731552672,
     "B": 3.1083596840941525,
     "C": 2.8248387024671335
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18979715087916701,
     "p50_s_raw": 0.01989857543958351,
     "p95_s_adjusted": 0.19419185474980621,
     "p95_s_raw": 0.022095927374903113
    },
    "views": {
     "Balanced 33:33:33": 3.5006149102123802,
     "Emphasis on Accuracy 60:20:20": 2.372543895109552,
     "Emphasis on Cost 20:20:60": 4.503957993600416,
     "Emphasis on Speed 20:60:20": 4.684348850018089,
     "Intelligence only": 1.599423855434685
    }
   },
   "rank": 208,
   "ranked": true,
   "ranks": {
    "A": 208,
    "B": 210,
    "C": 209,
    "capability": 206
   },
   "repo": "https://huggingface.co/eljiwo/jiwo-0.8b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 4.067693241125661,
    "B": 3.108324886935502,
    "C": 2.8247869730039317
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1898998890304938,
    "p50_s_raw": 0.019949944515246898,
    "p95_s_adjusted": 0.19480968727730213,
    "p95_s_raw": 0.022404843638651074
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5172413793103449,
       "cc": 24.45957255405951,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 0.0,
        "score": 6.712050995511854
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 24.45957255405951
      },
      "calibration_gen": {
       "accuracy": 0.35714285714285715,
       "cc": -9.724218679588965,
       "cc_by_type": {
        "choice": 25.46583850931677,
        "noul": -61.53846153846154,
        "score": 6.89996699037787
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.37755102040816324,
       "cc": -5.9291709848934415,
       "cc_by_type": {
        "choice": 40.74074074074074,
        "noul": -78.57142857142857,
        "score": 20.04317487600751
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.7777777777777778,
       "cc": 71.6287638691846,
       "cc_by_type": {
        "choice": 64.70588235294117,
        "noul": 100.0,
        "score": 50.1804092546126
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 71.6287638691846
      },
      "jaggedness": {
       "accuracy": 0.24324324324324326,
       "cc": -31.490478394830276,
       "cc_by_type": {
        "choice": -0.04631773969430275,
        "noul": -94.87179487179486,
        "score": 0.446677426998332
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.48863636363636365,
       "cc": 31.193268142476445,
       "cc_by_type": {
        "choice": 28.471899674872283,
        "noul": 12.328767123287676,
        "score": 52.77913762926938
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 31.193268142476445
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.16129032258064516,
       "cc": -35.935636537648094,
       "cc_by_type": {
        "choice": -1.5384615384615408,
        "noul": -100.0,
        "score": -6.268448074482746
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.37142857142857144,
       "cc": -0.3712463104306612,
       "cc_by_type": {
        "choice": 29.243170622480974,
        "noul": -47.12643678160919,
        "score": 16.769527227836235
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.35714285714285715,
       "cc": -7.457195719833385,
       "cc_by_type": {
        "choice": 14.285714285714286,
        "noul": -40.0,
        "score": 3.3426985547855614
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.49122807017543857,
       "cc": 35.62708102108768,
       "cc_by_type": {
        "choice": 35.62708102108768
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 35.62708102108768
      },
      "perturb_pair": {
       "accuracy": 0.6,
       "cc": 31.852412487051,
       "cc_by_type": {
        "choice": 52.81782437745741,
        "noul": -14.28571428571429,
        "score": 57.02512736940989
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 31.852412487051
      },
      "usecase": {
       "accuracy": 0.29957805907172996,
       "cc": -18.97090524037614,
       "cc_by_type": {
        "choice": 29.46617845442283,
        "noul": -82.45614035087719,
        "score": -3.9227538246740634
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -0.15643741530528152,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -50.0,
        "score": 29.530687754084152
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": -12.711864406779661,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.2916666666666667,
       "cc": 0.2625190049768242,
       "cc_by_type": {
        "choice": 5.325443786982243,
        "noul": -19.999999999999996,
        "score": 15.462113227948226
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0.2625190049768242
      },
      "de": {
       "accuracy": 0.4222222222222222,
       "cc": 6.827684779744186,
       "cc_by_type": {
        "choice": 29.390464695232346,
        "noul": -11.111111111111116,
        "score": 2.2037007551113263
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 6.827684779744186
      },
      "el": {
       "accuracy": 0.16666666666666666,
       "cc": -54.072622649773365,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100.0,
        "score": -65.83232578064539
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.3735478105451296,
       "cc": -4.109282546744619,
       "cc_by_type": {
        "choice": 25.29444756029158,
        "noul": -49.62406015037594,
        "score": 12.001764949850502
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.6071428571428571,
       "cc": 34.262221513293575,
       "cc_by_type": {
        "choice": 86.90909090909093,
        "noul": 9.090909090909083,
        "score": 6.786664539880705
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 34.262221513293575
      },
      "fi": {
       "accuracy": 0.3333333333333333,
       "cc": -31.690140845070424,
       "cc_by_type": {
        "choice": 36.61971830985915,
        "noul": -100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.4,
       "cc": -5.790035284708807,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": -100.0,
        "score": 33.405634625422245
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.06666666666666667,
       "cc": -36.64150500544096,
       "cc_by_type": {
        "choice": -24.22360248447205,
        "noul": -100.0,
        "score": 14.299087468149175
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.3333333333333333,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 55.395640882380825,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 50.0,
        "score": 66.96266312669114
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 55.395640882380825
      },
      "ja": {
       "accuracy": 0.21875,
       "cc": -28.34012514670054,
       "cc_by_type": {
        "choice": 10.569105691056912,
        "noul": -100.0,
        "score": 4.410518868841473
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.2727272727272727,
       "cc": -19.50727275296486,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": 7.0519522329086985
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.6363636363636364,
       "cc": 40.89245662958265,
       "cc_by_type": {
        "choice": 78.02197802197803,
        "noul": 0.0,
        "score": 44.655391866769904
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 40.89245662958265
      },
      "nl": {
       "accuracy": 0.23076923076923078,
       "cc": -23.738139726914895,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": -33.333333333333336,
        "score": -37.60253431537792
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.14285714285714285,
       "cc": -49.18032786885246,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.45,
       "cc": 25.233491597647614,
       "cc_by_type": {
        "choice": 60.64713598600787,
        "noul": 0.0,
        "score": 15.05333880693498
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 25.233491597647614
      },
      "pt": {
       "accuracy": 0.7222222222222222,
       "cc": 59.348158787470005,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 44.711143029076695
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 59.348158787470005
      },
      "sv": {
       "accuracy": 0.0,
       "cc": -47.36188981647681,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -8.752336116097092
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 0.2881809678820299,
       "cc_by_type": {
        "choice": 21.052631578947373,
        "noul": -33.333333333333336,
        "score": 13.145244658032052
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.2881809678820299
      },
      "uk": {
       "accuracy": 0.45454545454545453,
       "cc": -12.288605896536685,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": 50.09070404952038
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.375,
       "cc": 10.800875659098933,
       "cc_by_type": {
        "choice": 2.702702702702704,
        "noul": 33.33333333333333,
        "score": -3.6334090587392343
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 10.800875659098933
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.43333333333333335,
       "cc": 26.58490240110555,
       "cc_by_type": {
        "choice": 26.58490240110555
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 26.58490240110555
      },
      "noul": {
       "accuracy": 0.264,
       "cc": -47.199999999999996,
       "cc_by_type": {
        "noul": -47.199999999999996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.37333333333333335,
       "cc": 13.164164091005937,
       "cc_by_type": {
        "score": 13.164164091005937
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 13.164164091005937
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.375,
       "cc": -24.495021091479135,
       "cc_by_type": {
        "choice": 30.807248764415156,
        "noul": -100.0,
        "score": -4.292312038852564
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -33.15167050235345,
       "cc_by_type": {
        "choice": 21.440261865793783,
        "noul": -100.0,
        "score": -20.895273372854128
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.3,
       "cc": -1.166880782233278,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -75.0,
        "score": 36.01548668555823
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.2692307692307692,
       "cc": -29.51868765932299,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100.0,
        "score": -4.218713580378597
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -19.515561208758683,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100.0,
        "score": -2.6460625082636247
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.2413793103448276,
       "cc": -22.650212849237985,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -60.0,
        "score": -13.933544530619923
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.34615384615384615,
       "cc": -8.626664926886974,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -75.0,
        "score": 14.337396523686897
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.20689655172413793,
       "cc": -43.06325104574836,
       "cc_by_type": {
        "choice": 4.950495049504946,
        "noul": -100.0,
        "score": -34.140248186750014
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.42857142857142855,
       "cc": 16.29704619319791,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -100.0,
        "score": 96.51018619864135
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 16.29704619319791
      },
      "knowledge_graphs": {
       "accuracy": 0.13043478260869565,
       "cc": -36.43239114303143,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100.0,
        "score": -8.53381465046834
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.24,
       "cc": -15.40910199821853,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -75.0,
        "score": -6.186655588151524
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.38461538461538464,
       "cc": 48.23720641132996,
       "cc_by_type": {
        "choice": 18.032786885245905,
        "noul": 100.0,
        "score": 26.678832348743953
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 48.23720641132996
      },
      "llm_guardrails": {
       "accuracy": 0.3103448275862069,
       "cc": -8.538148364008457,
       "cc_by_type": {
        "choice": 44.95412844036697,
        "noul": -75.0,
        "score": 4.431426467607658
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.5133333333333333,
       "cc": 32.82106671729687,
       "cc_by_type": {
        "choice": 42.106673813990895,
        "noul": 9.090909090909083,
        "score": 47.265617246990644
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 32.82106671729687
      },
      "moderation_trust_safety": {
       "accuracy": 0.1875,
       "cc": -18.741819385691944,
       "cc_by_type": {
        "choice": -10.526315789473685,
        "noul": -100.0,
        "score": 54.30085763239786
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.23809523809523808,
       "cc": -18.91047217979512,
       "cc_by_type": {
        "choice": 9.547738693467341,
        "noul": -100.0,
        "score": 33.7208447671473
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": 6.905355691048825,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -56.0134926752812
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 6.905355691048825
      },
      "scientific_discovery": {
       "accuracy": 0.25,
       "cc": -23.30406902667455,
       "cc_by_type": {
        "choice": 20.31872509960159,
        "noul": -100.0,
        "score": 9.769067820374755
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.2727272727272727,
       "cc": -24.90976631788278,
       "cc_by_type": {
        "choice": 28.40095465393794,
        "noul": -100.0,
        "score": -3.1302536075862752
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.49837033035136,
      80.76387849103644
     ],
     "I": [
      12.343340329004636,
      18.854234371958732
     ],
     "capability": [
      43.298750810210294,
      49.02128383708443
     ],
     "composite": {
      "A": [
       2.0717818904784333,
       6.329209242448164
      ],
      "B": [
       1.5337127235876,
       4.973633390507989
      ],
      "C": [
       1.4387374239433561,
       4.3952841961445595
      ]
     },
     "seed": 22
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 78.54210067220151,
     "capability": 47.20710280189251,
     "intelligence": 15.872104931583502
    },
    "per_type": {
     "choice": {
      "accuracy": 0.43333333333333335,
      "calibration": 71.36245954848286,
      "cc": 26.58490240110555,
      "cc_by_type": {
       "choice": 26.58490240110555
      },
      "cc_tiered": 29.996434046841067,
      "cc_tiered_open": 35.509112308155075,
      "cc_tiered_sealed": 24.483755785527055,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 26.58490240110555
     },
     "noul": {
      "accuracy": 0.264,
      "calibration": 77.24881325795822,
      "cc": -47.199999999999996,
      "cc_by_type": {
       "noul": -47.199999999999996
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.37333333333333335,
      "calibration": 87.01502921016349,
      "cc": 13.164164091005937,
      "cc_by_type": {
       "score": 13.164164091005937
      },
      "cc_tiered": 17.61988074790945,
      "cc_tiered_open": 26.09045715280502,
      "cc_tiered_sealed": 9.149304343013874,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 13.164164091005937
     }
    },
    "raw_on_A": {
     "calibration": 73.78928446595333,
     "capability": 44.008100936829386,
     "intelligence": 14.22691740770544
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.5005413548379267,
    "Emphasis on Accuracy 60:20:20": 2.3725236224413906,
    "Emphasis on Cost 20:20:60": 4.503884935608954,
    "Emphasis on Speed 20:60:20": 4.684111776063039,
    "Intelligence only": 1.599423855434685
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "eljiwo / jiwidi (fhjaime96@gmail.com)",
   "axes": {
    "calibration": 85.66971124396188,
    "cost": 64.0899740452639,
    "intelligence": 34.784391844693545,
    "speed": 92.17972487720941
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0747562710407059,
      "ece_score": 85.04874579185882,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 85.04874579185882,
      "score_before_time_limit": 85.04874579185882,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.37362637362637363,
      "brier": 0.1479946142436308,
      "ece": 0.06661725466202273,
      "n": 364,
      "score": 86.67654906759546,
      "score_before_time_limit": 86.67654906759546,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9402173913043478,
      "interval90_width": 0.48007246376811585,
      "n": 368,
      "nrps": 0.1073762318235206,
      "score": 85.28383887243137,
      "score_before_time_limit": 85.28383887243137,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09347349536392596
     }
    },
    "score": 85.66971124396188
   },
   "capability": 60.227051544327715,
   "composite_ci95": {
    "A": [
     18.621368124842647,
     39.06792387987791
    ],
    "B": [
     15.790485865253082,
     34.9225612266599
    ],
    "C": [
     12.931505642251835,
     27.130502694359652
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line.; reference `deepinfra:Qwen/Qwen3.5-4B` frozen at the 25 Sep cut-off, deprecated afterwards (replaced by `Qwen/Qwen3.5-9B`); re-score if it changes (I-3) OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 786,993 input and 0 output tokens over 1,500 decisions. Estimated, not charged.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01573986
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jiwo 4b (Qwen3.5-4B backbone with a trained decision head, one forward pass, native probabilities, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own install line from their model card (pip install 'jiwo[cuda] @ git+https://github.com/jiwidi/jiwo@8716abab') and their own `jiwo serve`, nothing patched. The server reads JIWO_CHECKPOINT and calls from_pretrained with NO revision and there is no JIWO_REVISION setting, so it was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY and the served revision cannot have drifted to main. JIWO_MAX_LENGTH=65536, the author's own documented value for their published runs: the 8,192 default refuses longer prompts with HTTP 400 and never truncates. The trained context is 8,192 tokens, so the longest items of this draw are answered outside the range the model saw in training. CUDA graphs are OFF (JIWO_CUDA_GRAPHS unset): the author's own card says graphs agree with eager on only 99.7 % of fields, so eager is the exact path and the author's 14.5 ms headline uses a mode this measurement does not use. Requests are serialised by the author's own asyncio.Lock. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.202737997549946,
    "I_sealed": 28.366045691837137,
    "base": 34.784391844693545,
    "excess": 2.763447724373096,
    "gap": 12.836692305712809,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.6558109869045,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864866,
       "hard": 58.47750865051904,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 26.298826777087655,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.653576228657684,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.10967244172717,
       "hard": 38.322559765790174,
       "judge": 32.507251953307225,
       "standard": 27.80704746088366
      }
     },
     "sealed|choice": {
      "cc": 51.626839698704124,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 64.33413286056941,
       "hard": 48.059799934044186,
       "judge": 51.60062909035564,
       "standard": 52.446588559614064
      }
     },
     "sealed|noul": {
      "cc": 4.118819776714509,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 33.33333333333333,
       "hard": -7.57575757575758,
       "judge": 13.888888888888884,
       "standard": -1.7543859649122862
      }
     },
     "sealed|score": {
      "cc": 29.352477600092772,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 53.37012049696441,
       "hard": 14.78666626313042,
       "judge": 38.03240184856901,
       "standard": 33.455392452867294
      }
     }
    }
   },
   "jevbench_score": 28.949425698879754,
   "key": "jiwo-4b",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (weights, model card); MIT (code, github.com/jiwidi/jiwo)",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "eljiwo/jiwo-4b@9adc559ff7ec5baffade3998f7bca85da0d124ed",
   "noul_decisive": {
    "acc_among_decisive": 0.8991228070175439,
    "decisive": 228,
    "decisive_rate": 0.6263736263736264,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.66971124396188,
     "cost": 64.0899740452639,
     "intelligence": 34.784391844693545,
     "speed": 92.44547609658223
    },
    "board_release": "v1.6.43",
    "capability": 60.227051544327715,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 60,
    "capability_rank": 85,
    "composite_ci95": {
     "A": [
      18.62937312636143,
      39.08706741447612
     ],
     "B": [
      15.795090114565683,
      34.93479875570867
     ],
     "C": [
      12.937064671084322,
      27.143796815608415
     ]
    },
    "jevbench_score": 28.962932355698086,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 71,
    "rank": 103,
    "ranks": {
     "A": 103,
     "B": 104,
     "C": 113,
     "capability": 131
    },
    "scores": {
     "A": 28.962932355698086,
     "B": 25.315480694857687,
     "C": 20.113147469234782
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21877912301570177,
     "p50_s_raw": 0.034389561507850885,
     "p95_s_adjusted": 0.26028342521749437,
     "p95_s_raw": 0.05514171260874719
    },
    "views": {
     "Balanced 33:33:33": 26.318232161775516,
     "Emphasis on Accuracy 60:20:20": 21.478631369812923,
     "Emphasis on Cost 20:20:60": 28.0163293235414,
     "Emphasis on Speed 20:60:20": 31.50792357388672,
     "Intelligence only": 16.835004451338282
    }
   },
   "rank": 118,
   "ranked": true,
   "ranks": {
    "A": 118,
    "B": 119,
    "C": 131,
    "capability": 149
   },
   "repo": "https://huggingface.co/eljiwo/jiwo-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 28.949425698879754,
    "B": 25.30722438347579,
    "C": 20.10376784644427
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.222632587980479,
    "p50_s_raw": 0.0363162939902395,
    "p95_s_adjusted": 0.27191851355601104,
    "p95_s_raw": 0.06095925677800551
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7586206896551724,
       "cc": 63.41069040078068,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 66.66666666666667,
        "score": 56.8987378690087
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 63.41069040078068
      },
      "calibration_gen": {
       "accuracy": 0.5446428571428571,
       "cc": 21.306873408130947,
       "cc_by_type": {
        "choice": 51.08695652173913,
        "noul": 0.0,
        "score": 12.833663702653709
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 21.306873408130947
      },
      "composite_gen": {
       "accuracy": 0.6020408163265306,
       "cc": 40.276766724193266,
       "cc_by_type": {
        "choice": 62.96296296296296,
        "noul": 21.42857142857142,
        "score": 36.438765781045426
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 40.276766724193266
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 97.09800473660214,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 91.29401420980636
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 97.09800473660214
      },
      "jaggedness": {
       "accuracy": 0.372972972972973,
       "cc": -11.082947542546755,
       "cc_by_type": {
        "choice": 29.133858267716537,
        "noul": -69.23076923076923,
        "score": 6.848068335412427
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6893939393939394,
       "cc": 58.2269224408714,
       "cc_by_type": {
        "choice": 51.69530887134232,
        "noul": 50.68493150684932,
        "score": 72.30052694442254
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 58.2269224408714
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3870967741935484,
       "cc": -16.440841671706764,
       "cc_by_type": {
        "choice": 16.923076923076923,
        "noul": -75.0,
        "score": 8.754398061802782
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6158730158730159,
       "cc": 34.72665534729893,
       "cc_by_type": {
        "choice": 57.00850873264667,
        "noul": 17.24137931034482,
        "score": 29.930077998905315
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 34.72665534729893
      },
      "negation_pair_gen": {
       "accuracy": 0.5892857142857143,
       "cc": 2.0857022253018394,
       "cc_by_type": {
        "choice": 57.142857142857146,
        "noul": 10.000000000000009,
        "score": -60.885750466951635
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 2.0857022253018394
      },
      "option_order": {
       "accuracy": 0.9122807017543859,
       "cc": 88.90122086570477,
       "cc_by_type": {
        "choice": 88.90122086570477
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 88.90122086570477
      },
      "perturb_pair": {
       "accuracy": 0.85,
       "cc": 73.88280168114402,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 57.14285714285714,
        "score": 80.23293977475579
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 73.88280168114402
      },
      "usecase": {
       "accuracy": 0.5485232067510548,
       "cc": 27.448440267073206,
       "cc_by_type": {
        "choice": 51.4357294276354,
        "noul": 1.7543859649122862,
        "score": 29.155205408671925
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 27.448440267073206
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -16.233054801263584,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -50.0,
        "score": -18.699164403790757
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 28.531073446327685,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 28.531073446327685
      },
      "da": {
       "accuracy": 0.5833333333333334,
       "cc": 40.559189285398894,
       "cc_by_type": {
        "choice": 40.8284023668639,
        "noul": 19.999999999999996,
        "score": 60.84916548933277
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 40.559189285398894
      },
      "de": {
       "accuracy": 0.6888888888888889,
       "cc": 43.45157490330706,
       "cc_by_type": {
        "choice": 72.84248642124321,
        "noul": 33.33333333333333,
        "score": 24.178904955344628
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 43.45157490330706
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": -27.556284563855588,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -60.0,
        "score": -62.42788983614507
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.5889186773905273,
       "cc": 30.320127791507446,
       "cc_by_type": {
        "choice": 53.7857543466068,
        "noul": 5.263157894736836,
        "score": 31.911471133178704
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 30.320127791507446
      },
      "es": {
       "accuracy": 0.75,
       "cc": 53.31121705049497,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 45.45454545454546,
        "score": 40.660923878757615
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 53.31121705049497
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 24.4131455399061,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 24.4131455399061
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 19.936766375030352,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -50.0,
        "score": 35.19816936486537
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 19.936766375030352
      },
      "hi": {
       "accuracy": 0.43333333333333335,
       "cc": 12.211958226018675,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": 0.0,
        "score": 11.170036168739262
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 12.211958226018675
      },
      "id": {
       "accuracy": 0.8333333333333334,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 50.0
      },
      "it": {
       "accuracy": 0.8235294117647058,
       "cc": 65.35071606453869,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 50.0,
        "score": 46.05214819361604
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 65.35071606453869
      },
      "ja": {
       "accuracy": 0.4375,
       "cc": 12.03476070227424,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -14.28571428571429,
        "score": 15.430646799041082
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 12.03476070227424
      },
      "ko": {
       "accuracy": 0.6363636363636364,
       "cc": 57.974720722270284,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 6.711047412712501
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 57.974720722270284
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 89.40690554679048,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 68.22071664037144
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 89.40690554679048
      },
      "nl": {
       "accuracy": 0.5384615384615384,
       "cc": 30.176686204383895,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0.0,
        "score": 40.66933437916839
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 30.176686204383895
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.6,
       "cc": 33.93999941140274,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 0.0,
        "score": 25.431716642603497
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 33.93999941140274
      },
      "pt": {
       "accuracy": 0.8888888888888888,
       "cc": 77.91153300960076,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 66.66666666666667,
        "score": 85.15838462344217
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 77.91153300960076
      },
      "sv": {
       "accuracy": 0.375,
       "cc": -8.765750483399948,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 50.0,
        "score": -42.96391811686651
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 24.254058362470982,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -34.25536877223618
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 24.254058362470982
      },
      "uk": {
       "accuracy": 0.45454545454545453,
       "cc": -18.250772299457207,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": 32.20420484075882
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 61.21814460262698,
       "cc_by_type": {
        "choice": 67.56756756756756,
        "noul": 33.33333333333333,
        "score": 82.75353290698004
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 61.21814460262698
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6453333333333333,
       "cc": 54.05078597339782,
       "cc_by_type": {
        "choice": 54.05078597339782
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 54.05078597339782
      },
      "noul": {
       "accuracy": 0.5466666666666666,
       "cc": 9.333333333333327,
       "cc_by_type": {
        "noul": 9.333333333333327
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 9.333333333333327
      },
      "score": {
       "accuracy": 0.5413333333333333,
       "cc": 31.101047707694065,
       "cc_by_type": {
        "score": 31.101047707694065
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 31.101047707694065
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5833333333333334,
       "cc": 24.071004328629304,
       "cc_by_type": {
        "choice": 40.691927512355846,
        "noul": -19.999999999999996,
        "score": 51.52108547353207
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 24.071004328629304
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 39.264007955407884,
       "cc_by_type": {
        "choice": 41.08019639934533,
        "noul": 60.00000000000001,
        "score": 16.711827466878294
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 39.264007955407884
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 20.00867761558955,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -25.0,
        "score": 49.54216187902672
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 20.00867761558955
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": -10.818142384890914,
       "cc_by_type": {
        "choice": 27.710843373493972,
        "noul": -50.0,
        "score": -10.165270528166715
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 24.55248819664185,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -19.999999999999996,
        "score": 30.92454533526716
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 24.55248819664185
      },
      "feature_extraction": {
       "accuracy": 0.4482758620689655,
       "cc": 12.840294527779639,
       "cc_by_type": {
        "choice": 23.076923076923077,
        "noul": -19.999999999999996,
        "score": 35.44396050641584
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 12.840294527779639
      },
      "financial_crime": {
       "accuracy": 0.6153846153846154,
       "cc": 42.181818059125035,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": 75.0,
        "score": 3.7193672208533535
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 42.181818059125035
      },
      "gaming": {
       "accuracy": 0.5172413793103449,
       "cc": 15.436535058528824,
       "cc_by_type": {
        "choice": 52.475247524752476,
        "noul": -14.28571428571429,
        "score": 8.120071936548289
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 15.436535058528824
      },
      "insurance_claims": {
       "accuracy": 0.5,
       "cc": 2.555674036706544,
       "cc_by_type": {
        "choice": 68.25396825396825,
        "noul": -60.0,
        "score": -0.5869461438486212
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 2.555674036706544
      },
      "knowledge_graphs": {
       "accuracy": 0.4782608695652174,
       "cc": 14.125622525737157,
       "cc_by_type": {
        "choice": 35.87786259541985,
        "noul": -19.999999999999996,
        "score": 26.499004981791618
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 14.125622525737157
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 11.047439957865064,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -50.0,
        "score": -0.59751752477879
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 11.047439957865064
      },
      "legal_compliance": {
       "accuracy": 0.46153846153846156,
       "cc": -11.127349456353167,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": 32.19172212274377
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.6551724137931034,
       "cc": 53.749288065206805,
       "cc_by_type": {
        "choice": 63.30275229357799,
        "noul": 50.0,
        "score": 47.945111902042434
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 53.749288065206805
      },
      "model_routing": {
       "accuracy": 0.7133333333333334,
       "cc": 58.29415940370154,
       "cc_by_type": {
        "choice": 64.62074510855,
        "noul": 59.09090909090908,
        "score": 51.17082401164556
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 58.29415940370154
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 16.05980603069598,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": -60.0,
        "score": 55.547839144719525
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 16.05980603069598
      },
      "recruiting": {
       "accuracy": 0.6190476190476191,
       "cc": 33.863204600776136,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 0.0,
        "score": 31.740367571172644
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 33.863204600776136
      },
      "risk_assessment": {
       "accuracy": 0.3076923076923077,
       "cc": -16.123855817090945,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -33.333333333333336,
        "score": -20.698611476430063
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.375,
       "cc": -7.830166742245569,
       "cc_by_type": {
        "choice": 28.286852589641434,
        "noul": -71.42857142857143,
        "score": 19.65121861219329
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.8181818181818182,
       "cc": 66.53707551936073,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 66.66666666666667,
        "score": 47.26436896062795
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 66.53707551936073
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.08166368366025,
      86.75710341088941
     ],
     "I": [
      29.097201235740425,
      39.427338008588556
     ],
     "capability": [
      55.64492730946361,
      62.239450216352004
     ],
     "composite": {
      "A": [
       18.621368124842647,
       39.06792387987791
      ],
      "B": [
       15.790485865253082,
       34.9225612266599
      ],
      "C": [
       12.931505642251835,
       27.130502694359652
      ]
     },
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.66971124396188,
     "capability": 60.227051544327715,
     "intelligence": 34.784391844693545
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6453333333333333,
      "calibration": 85.04874579185882,
      "cc": 54.05078597339782,
      "cc_by_type": {
       "choice": 54.05078597339782
      },
      "cc_tiered": 56.64132534280431,
      "cc_tiered_open": 61.6558109869045,
      "cc_tiered_sealed": 51.626839698704124,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.05078597339782
     },
     "noul": {
      "accuracy": 0.5466666666666666,
      "calibration": 86.67654906759546,
      "cc": 9.333333333333327,
      "cc_by_type": {
       "noul": 9.333333333333327
      },
      "cc_tiered": 15.208823276901082,
      "cc_tiered_open": 26.298826777087655,
      "cc_tiered_sealed": 4.118819776714509,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 9.333333333333327
     },
     "score": {
      "accuracy": 0.5413333333333333,
      "calibration": 85.28383887243137,
      "cc": 31.101047707694065,
      "cc_by_type": {
       "score": 31.101047707694065
      },
      "cc_tiered": 32.503026914375226,
      "cc_tiered_open": 35.653576228657684,
      "cc_tiered_sealed": 29.352477600092772,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 31.101047707694065
     }
    },
    "raw_on_A": {
     "calibration": 82.12950185196465,
     "capability": 58.61658974199523,
     "intelligence": 35.1036776320258
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 26.30336349187957,
    "Emphasis on Accuracy 60:20:20": 21.472687786103617,
    "Emphasis on Cost 20:20:60": 28.006217696006477,
    "Emphasis on Speed 20:60:20": 31.469589278706508,
    "Intelligence only": 16.835004451338282
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 76.87411794897184,
    "cost": 50.56108621702079,
    "intelligence": 22.13919485880221,
    "speed": 94.0900145591257
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11313979352292604,
      "ece_score": 77.37204129541479,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 77.37204129541479,
      "score_before_time_limit": 77.37204129541479,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.5093333333333334,
      "brier": 0.19849004201230716,
      "ece": 0.11360395591990471,
      "n": 375,
      "score": 77.27920881601906,
      "score_before_time_limit": 77.27920881601906,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9093333333333333,
      "interval90_width": 0.5644444444444446,
      "n": 375,
      "nrps": 0.1509830793884296,
      "score": 75.9711037354817,
      "score_before_time_limit": 75.9711037354817,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.16479742295096816
     }
    },
    "score": 76.87411794897184
   },
   "capability": 49.50665640388703,
   "composite_ci95": {
    "A": [
     5.217354515822397,
     12.201151252166042
    ],
    "B": [
     4.192329175683889,
     10.346538617725733
    ],
    "C": [
     3.6231628582099975,
     8.473021702893085
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no bookable listing for this base. The declared base LiquidAI/LFM2.5-VL-3B is not a key of any frozen pricing table, and the frozen 25 Sep 2026 snapshot's only Liquid entry is liquid/lfm-2.5-2.6b:free at (0, 0) - a free route, which may not price a row (Florian, free tiers: use the published paid price or the estimate). Method, as aplomb-1, wald-4b-v2, jiwo-* and the ClassOne trio carry: the DeepInfra Qwen3.5 size-class reference from the frozen snapshot, rounded ONE CLASS UP so the estimate can only OVERSTATE this row's cost - deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output, for a 3B model. Applied to this row's own measured usage: 2,222,971 input tokens and 0 output tokens over 1,500 decisions. The 2B rate (0.02, 0.10) would give 0.029639613333333335 and was deliberately NOT used because it understates. Precedent for a LiquidAI base priced as a labelled estimate: the live lev-350m row (LiquidAI/LFM2.5-350M, in UNLISTED_BASE_MODELS, estimate 0.00462807881773399 per 1,000).",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.044459420000000006
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Liquid AI d1-3B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX 5090 32 GB pod (France), offline after setup (egress cut and proven), author's custom code at the pinned commit via trust_remote_code from the sha256-verified local copy, bfloat16, no torch.compile",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 27.85582362507926,
    "I_sealed": 16.42256609252516,
    "base": 22.13919485880221,
    "excess": 1.3600129512143884,
    "gap": 11.433257532554101,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 53.72008003683186,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675676,
       "hard": 43.378420887071414,
       "judge": 60.44825313117996,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -30.434782608695656,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 29.847390838405925,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.55757331890299,
       "hard": 17.69560694707587,
       "judge": 41.70520484649972,
       "standard": 33.00914636867683
      }
     },
     "sealed|choice": {
      "cc": 39.056033872257295,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 54.948378350192925,
       "hard": 33.21974277234252,
       "judge": 38.815889604789206,
       "standard": 43.14266023432116
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 7.692307692307687,
       "hard": -45.45454545454546,
       "judge": -5.555555555555558,
       "standard": -5.263157894736848
      }
     },
     "sealed|score": {
      "cc": 10.211664405318189,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 27.92356315993724,
       "hard": 9.442062390485162,
       "judge": 0.13958879484807785,
       "standard": 18.003032473379875
      }
     }
    }
   },
   "jevbench_score": 8.853035001087052,
   "key": "liquid-d1-3b",
   "last_measured_on": "2026-10-10",
   "licence": "LFM Open License v1.0 (commercial use only below USD 10 M annual revenue)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "LiquidAI/d1-3B@051bcc464b01b9f92942b364d9586b0ef5912432",
   "noul_decisive": {
    "acc_among_decisive": 0.8315217391304348,
    "decisive": 184,
    "decisive_rate": 0.49066666666666664,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 76.87411794897184,
     "cost": 50.56108621702079,
     "intelligence": 22.13919485880221,
     "speed": 94.32773191369529
    },
    "board_release": "v1.6.43",
    "capability": 49.50665640388703,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 91,
    "capability_rank": 118,
    "composite_ci95": {
     "A": [
      5.218755487589073,
      12.205081733672536
     ],
     "B": [
      4.193052691245517,
      10.348794476091657
     ],
     "C": [
      3.6241357552701894,
      8.475751203939263
     ]
    },
    "jevbench_score": 8.855712627780655,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 136,
    "rank": 175,
    "ranks": {
     "A": 175,
     "B": 176,
     "C": 176,
     "capability": 175
    },
    "scores": {
     "A": 8.855712627780655,
     "B": 7.330614913163864,
     "C": 6.149800435958788
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1797857349505648,
     "p50_s_raw": 0.0148928674752824,
     "p95_s_adjusted": 0.20533906437922267,
     "p95_s_raw": 0.027669532189611334
    },
    "views": {
     "Balanced 33:33:33": 7.785401938124825,
     "Emphasis on Accuracy 60:20:20": 5.909418623202946,
     "Emphasis on Cost 20:20:60": 8.516521034074662,
     "Emphasis on Speed 20:60:20": 10.132088187487657,
     "Intelligence only": 4.340556958274228
    }
   },
   "rank": 197,
   "ranked": true,
   "ranks": {
    "A": 197,
    "B": 198,
    "C": 198,
    "capability": 195
   },
   "repo": "https://huggingface.co/LiquidAI/d1-3B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 8.853035001087052,
    "B": 7.329146939870814,
    "C": 6.14794097297712
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18326355200260877,
    "p50_s_raw": 0.016631776001304388,
    "p95_s_adjusted": 0.21277590398676693,
    "p95_s_raw": 0.03138795199338346
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7241379310344828,
       "cc": 54.390853941466354,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 66.66666666666667,
        "score": 21.50589515773239
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 54.390853941466354
      },
      "calibration_gen": {
       "accuracy": 0.45535714285714285,
       "cc": 7.885435170467154,
       "cc_by_type": {
        "choice": 41.770186335403736,
        "noul": 0.0,
        "score": -18.113880824002273
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 7.885435170467154
      },
      "composite_gen": {
       "accuracy": 0.5408163265306123,
       "cc": 25.303042500575433,
       "cc_by_type": {
        "choice": 77.77777777777779,
        "noul": -21.42857142857143,
        "score": 19.559921152519944
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 25.303042500575433
      },
      "fairness": {
       "accuracy": 0.8518518518518519,
       "cc": 47.39465841291355,
       "cc_by_type": {
        "choice": 88.23529411764707,
        "noul": 0.0,
        "score": 53.948681121093585
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 47.39465841291355
      },
      "jaggedness": {
       "accuracy": 0.31891891891891894,
       "cc": -22.274181763566684,
       "cc_by_type": {
        "choice": 20.79666512274201,
        "noul": -79.48717948717949,
        "score": -8.13203092626258
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.625,
       "cc": 51.3034512100294,
       "cc_by_type": {
        "choice": 47.05062703204832,
        "noul": 50.68493150684932,
        "score": 56.17479509119057
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 51.3034512100294
      },
      "long_state": {
       "accuracy": 0.1724137931034483,
       "cc": -29.57896952664178,
       "cc_by_type": {
        "choice": 16.66666666666667,
        "noul": -100.0,
        "score": -5.403575246592007
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2903225806451613,
       "cc": -13.862612846690979,
       "cc_by_type": {
        "choice": 7.692307692307688,
        "noul": -50.0,
        "score": 0.7198537676193717
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.41904761904761906,
       "cc": 4.0623722132052436,
       "cc_by_type": {
        "choice": 34.617107030900144,
        "noul": -33.333333333333336,
        "score": 10.903342942048921
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 4.0623722132052436
      },
      "negation_pair_gen": {
       "accuracy": 0.44642857142857145,
       "cc": -3.4722633613639875,
       "cc_by_type": {
        "choice": 38.095238095238095,
        "noul": -50.0,
        "score": 1.4879718206699422
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.7017543859649122,
       "cc": 62.264150943396224,
       "cc_by_type": {
        "choice": 62.264150943396224
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 62.264150943396224
      },
      "perturb_pair": {
       "accuracy": 0.7166666666666667,
       "cc": 57.31023460546714,
       "cc_by_type": {
        "choice": 52.81782437745741,
        "noul": 42.85714285714286,
        "score": 76.25573658180112
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 57.31023460546714
      },
      "usecase": {
       "accuracy": 0.39662447257383965,
       "cc": -1.9444617361056398,
       "cc_by_type": {
        "choice": 38.716515706301806,
        "noul": -43.85964912280702,
        "score": -0.6902517918117024
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.18181818181818182,
       "cc": -100.14332542233501,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -100.0,
        "score": -220.429976267005
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.375,
       "cc": -4.802259887005647,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.375,
       "cc": -8.98770932887382,
       "cc_by_type": {
        "choice": 17.159763313609464,
        "noul": -60.0,
        "score": 15.877108699769082
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.4222222222222222,
       "cc": 6.322815228633009,
       "cc_by_type": {
        "choice": 34.8219674109837,
        "noul": -11.111111111111116,
        "score": -4.742410613973558
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 6.322815228633009
      },
      "el": {
       "accuracy": 0.2777777777777778,
       "cc": -39.39660414361751,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100.0,
        "score": -33.85246303326217
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.4924039320822163,
       "cc": 14.003345625582787,
       "cc_by_type": {
        "choice": 44.13909141895677,
        "noul": -18.045112781954884,
        "score": 15.916058239746478
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 14.003345625582787
      },
      "es": {
       "accuracy": 0.6428571428571429,
       "cc": 33.758516235110754,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 27.27272727272727,
        "score": 0.18463961442317656
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 33.758516235110754
      },
      "fi": {
       "accuracy": 0.0,
       "cc": -63.38028169014085,
       "cc_by_type": {
        "choice": -26.760563380281692,
        "noul": -100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 28.727877312438604,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -50.0,
        "score": 36.183631937315795
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 28.727877312438604
      },
      "hi": {
       "accuracy": 0.3,
       "cc": 1.3860870885139314,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": -19.999999999999996,
        "score": 11.114783004672224
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 1.3860870885139314
      },
      "id": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 17.741935483870968
      },
      "it": {
       "accuracy": 0.6470588235294118,
       "cc": 40.934769362003244,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 50.0,
        "score": 23.5800485655584
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 40.934769362003244
      },
      "ja": {
       "accuracy": 0.34375,
       "cc": -7.243137787326186,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -42.85714285714286,
        "score": 2.428542503294384
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.5454545454545454,
       "cc": 21.42666905349395,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 0.0,
        "score": 29.853777652285117
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 21.42666905349395
      },
      "mixed": {
       "accuracy": 0.7272727272727273,
       "cc": 49.321740609526785,
       "cc_by_type": {
        "choice": 78.02197802197803,
        "noul": 0.0,
        "score": 69.94324380660233
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 49.321740609526785
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": 19.828553848785166,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 0.0,
        "score": 59.76421307838892
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 19.828553848785166
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 17.486338797814206,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 17.486338797814206
      },
      "pl": {
       "accuracy": 0.525,
       "cc": 29.992507104770652,
       "cc_by_type": {
        "choice": 84.25885439440316,
        "noul": 0.0,
        "score": 5.718666919908799
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 29.992507104770652
      },
      "pt": {
       "accuracy": 0.6666666666666666,
       "cc": 54.789361628198456,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": 33.33333333333333,
        "score": 67.2156560738751
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 54.789361628198456
      },
      "sv": {
       "accuracy": 0.0,
       "cc": -60.92596627415683,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -49.44456548913716
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": -25.14391404022173,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": -33.333333333333336,
        "score": -115.78261931364766
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": -3.966886884613037,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100.0,
        "score": 53.3167306505087
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 27.7843293385466,
       "cc_by_type": {
        "choice": 2.702702702702704,
        "noul": 33.33333333333333,
        "score": 47.31695197960376
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 27.7843293385466
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5546666666666666,
       "cc": 42.30437035757471,
       "cc_by_type": {
        "choice": 42.30437035757471
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 42.30437035757471
      },
      "noul": {
       "accuracy": 0.408,
       "cc": -18.400000000000006,
       "cc_by_type": {
        "noul": -18.400000000000006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.4026666666666667,
       "cc": 14.785081876637218,
       "cc_by_type": {
        "score": 14.785081876637218
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 14.785081876637218
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4166666666666667,
       "cc": -21.322066560908976,
       "cc_by_type": {
        "choice": 50.57660626029654,
        "noul": -60.0,
        "score": -54.542805943023474
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -20.883079422436126,
       "cc_by_type": {
        "choice": 1.8003273322422266,
        "noul": -19.999999999999996,
        "score": -44.4495655995506
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.55,
       "cc": 24.46389210511948,
       "cc_by_type": {
        "choice": 83.87096774193549,
        "noul": -50.0,
        "score": 39.52070857342296
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 24.46389210511948
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -33.81567016039666,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100.0,
        "score": 6.986724458569061
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.3888888888888889,
       "cc": 4.454876806713197,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -60.0,
        "score": 29.265251538152015
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 4.454876806713197
      },
      "feature_extraction": {
       "accuracy": 0.3103448275862069,
       "cc": -6.573360507662842,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -19.999999999999996,
        "score": -5.702987505894508
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.46153846153846156,
       "cc": 16.612884611673458,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": 25.0,
        "score": -22.987433121501375
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 16.612884611673458
      },
      "gaming": {
       "accuracy": 0.41379310344827586,
       "cc": -6.703885975467817,
       "cc_by_type": {
        "choice": 40.59405940594059,
        "noul": -71.42857142857143,
        "score": 10.722854096227385
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.21428571428571427,
       "cc": -33.44343074859378,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -100.0,
        "score": -20.965212880701966
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.17391304347826086,
       "cc": -31.072633553703692,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100.0,
        "score": 7.5454581175148805
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.32,
       "cc": -1.078959073185669,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -75.0,
        "score": 4.283447983695021
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.15384615384615385,
       "cc": -23.678295476921054,
       "cc_by_type": {
        "choice": -14.754098360655735,
        "noul": -100.0,
        "score": 43.719211929892566
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.4827586206896552,
       "cc": 15.868347401942737,
       "cc_by_type": {
        "choice": 35.77981651376147,
        "noul": 25.0,
        "score": -13.174774307933257
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 15.868347401942737
      },
      "model_routing": {
       "accuracy": 0.6866666666666666,
       "cc": 57.504158219489966,
       "cc_by_type": {
        "choice": 63.012597158938625,
        "noul": 59.09090909090908,
        "score": 50.40896840862219
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 57.504158219489966
      },
      "moderation_trust_safety": {
       "accuracy": 0.4375,
       "cc": 7.985104053150214,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -60.0,
        "score": 47.113206896292745
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 7.985104053150214
      },
      "recruiting": {
       "accuracy": 0.38095238095238093,
       "cc": 0.5006746462399576,
       "cc_by_type": {
        "choice": 39.69849246231156,
        "noul": -66.66666666666667,
        "score": 28.470198143074978
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0.5006746462399576
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": 6.972667050442593,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": -33.333333333333336,
        "score": 10.855108069566777
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 6.972667050442593
      },
      "scientific_discovery": {
       "accuracy": 0.25,
       "cc": -22.282698718421653,
       "cc_by_type": {
        "choice": 12.350597609561751,
        "noul": -71.42857142857143,
        "score": -7.77012233625527
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.45454545454545453,
       "cc": -9.604825720085183,
       "cc_by_type": {
        "choice": 57.04057279236276,
        "noul": -100.0,
        "score": 14.144950047381688
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.92554914167138,
      79.78637653206756
     ],
     "I": [
      18.019293258402904,
      25.188069370363653
     ],
     "capability": [
      45.240972947803556,
      51.6024129927801
     ],
     "composite": {
      "A": [
       5.217354515822397,
       12.201151252166042
      ],
      "B": [
       4.192329175683889,
       10.346538617725733
      ],
      "C": [
       3.6231628582099975,
       8.473021702893085
      ]
     },
     "seed": 24
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 76.87411794897184,
     "capability": 49.50665640388703,
     "intelligence": 22.13919485880221
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5546666666666666,
      "calibration": 77.37204129541479,
      "cc": 42.30437035757471,
      "cc_by_type": {
       "choice": 42.30437035757471
      },
      "cc_tiered": 46.38805695454458,
      "cc_tiered_open": 53.72008003683186,
      "cc_tiered_sealed": 39.056033872257295,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 42.30437035757471
     },
     "noul": {
      "accuracy": 0.408,
      "calibration": 77.27920881601906,
      "cc": -18.400000000000006,
      "cc_by_type": {
       "noul": -18.400000000000006
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.4026666666666667,
      "calibration": 75.9711037354817,
      "cc": 14.785081876637218,
      "cc_by_type": {
       "score": 14.785081876637218
      },
      "cc_tiered": 20.02952762186206,
      "cc_tiered_open": 29.847390838405925,
      "cc_tiered_sealed": 10.211664405318189,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 14.785081876637218
     }
    },
    "raw_on_A": {
     "calibration": 72.6387411013128,
     "capability": 46.47345221333068,
     "intelligence": 20.308163325348563
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 7.7826427506909805,
    "Emphasis on Accuracy 60:20:20": 5.908464633435587,
    "Emphasis on Cost 20:20:60": 8.514539745568566,
    "Emphasis on Speed 20:60:20": 10.123680376098372,
    "Intelligence only": 4.340556958274228
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 62.32062493293682,
    "cost": 73.1039158657426,
    "intelligence": 3.2615852143588597,
    "speed": 93.99244100826726
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22880094970871764,
      "ece_score": 54.23981005825647,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 54.23981005825647,
      "score_before_time_limit": 54.23981005825647,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.7333333333333334,
      "brier": 0.2769707015707441,
      "ece": 0.1533565516303061,
      "n": 375,
      "score": 69.32868967393878,
      "score_before_time_limit": 69.32868967393878,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8293333333333334,
      "interval90_width": 0.6586666666666664,
      "n": 375,
      "nrps": 0.21825554880108286,
      "score": 63.39337506661522,
      "score_before_time_limit": 63.39337506661522,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.25693847493330635
     }
    },
    "score": 62.32062493293682
   },
   "capability": 32.79110507364784,
   "composite_ci95": {
    "A": [
     0.007928751847345473,
     0.33896544733805034
    ],
    "B": [
     0.005122646508709089,
     0.23614958881345818
    ],
    "C": [
     0.005506077671767688,
     0.23539267176253492
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no bookable listing for this base. The declared text base LiquidAI/LFM2.5-Encoder-350M is not a key of any frozen pricing table. Method as above: deepinfra:Qwen/Qwen3.5-0.8B at USD 0.01 per 1M input and USD 0.05 per 1M output - the class above this 600M model - applied to its own measured usage, 1,182,027 input tokens and 0 output tokens over 1,500 decisions. The frozen base-size ENCODER rate (0.005, 0.0), which the published tacet-sonata and lev-350m rows carry, would give 0.00394009 and was deliberately NOT used: it prices a text-only encoder, while d1-omni also runs SigLIP2 vision and FastConformer audio towers, so the class above is the conservative choice and keeps the two Liquid rows on one method.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00788018
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Liquid AI d1-omni-600M",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (United States), offline after setup (egress cut and proven), author's custom code at the pinned commit via trust_remote_code from the sha256-verified local copy, float16, states cut on the right at 16,384 tokens by the model code",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.293220672965477,
    "I_sealed": 2.229949755752243,
    "base": 3.2615852143588597,
    "excess": -8.009973664126479,
    "gap": 2.063270917213234,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 12.87966201889643,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486504,
       "hard": 3.743315508021402,
       "judge": 12.986156888595916,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -73.91304347826086,
       "judge": -100.0,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 12.092551202372192,
       "hard": -10.469003850745073,
       "judge": 19.089799023912114,
       "standard": -20.24343611383641
      }
     },
     "sealed|choice": {
      "cc": 6.689849267256729,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 2.388153092084683,
       "hard": 3.5396284489392005,
       "judge": 5.9408452133326675,
       "standard": 16.2646450723639
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -64.1025641025641,
       "hard": -72.72727272727273,
       "judge": -72.22222222222221,
       "standard": -64.91228070175438
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": -14.21064395948739,
       "hard": -26.382300409685588,
       "judge": -13.560363505639916,
       "standard": -9.691900455584058
      }
     }
    }
   },
   "jevbench_score": 0.04905613733213806,
   "key": "liquid-d1-omni-600m",
   "last_measured_on": "2026-10-10",
   "licence": "LFM Open License v1.0 (commercial use only below USD 10 M annual revenue)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "LiquidAI/d1-omni-600M@02b55d7076f15129e59ab3f94783f32c4b088674",
   "noul_decisive": {
    "acc_among_decisive": 0.56,
    "decisive": 100,
    "decisive_rate": 0.26666666666666666,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 62.32062493293682,
     "cost": 73.1039158657426,
     "intelligence": 3.2615852143588597,
     "speed": 94.0744546594334
    },
    "board_release": "v1.6.43",
    "capability": 32.79110507364784,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 126,
    "capability_rank": 155,
    "composite_ci95": {
     "A": [
      0.00792887171051229,
      0.3389815219491588
     ],
     "B": [
      0.005122686535585832,
      0.23615583025477924
     ],
     "C": [
      0.005506160910077979,
      0.23540383468691578
     ]
    },
    "jevbench_score": 0.049057448753865106,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 182,
    "rank": 226,
    "ranks": {
     "A": 226,
     "B": 226,
     "C": 226,
     "capability": 222
    },
    "scores": {
     "A": 0.049057448753865106,
     "B": 0.032554129042754114,
     "C": 0.03406767274573965
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19256855882704257,
     "p50_s_raw": 0.02128427941352129,
     "p95_s_adjusted": 0.20322126662358642,
     "p95_s_raw": 0.026610633311793207
    },
    "views": {
     "Balanced 33:33:33": 0.038577217040809395,
     "Emphasis on Accuracy 60:20:20": 0.022535446033063476,
     "Emphasis on Cost 20:20:60": 0.05938557623710964,
     "Emphasis on Speed 20:60:20": 0.06041396685797389,
     "Intelligence only": 0.013878616661020313
    }
   },
   "rank": 249,
   "ranked": true,
   "ranks": {
    "A": 249,
    "B": 249,
    "C": 249,
    "capability": 244
   },
   "repo": "https://huggingface.co/LiquidAI/d1-omni-600M",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.04905613733213806,
    "B": 0.032553667044800734,
    "C": 0.03406676203620698
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19325650818645954,
    "p50_s_raw": 0.02162825409322977,
    "p95_s_adjusted": 0.20635821893811224,
    "p95_s_raw": 0.028179109469056128
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.20689655172413793,
       "cc": -29.121942106010263,
       "cc_by_type": {
        "choice": -8.333333333333334,
        "noul": -100.0,
        "score": 20.967507015302544
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.1875,
       "cc": -48.670081675280734,
       "cc_by_type": {
        "choice": -0.1552795031055859,
        "noul": -76.92307692307692,
        "score": -68.93188859965969
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.22448979591836735,
       "cc": -44.09414082419346,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -78.57142857142857,
        "score": -64.82210501226291
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.2222222222222222,
       "cc": -39.3458024428534,
       "cc_by_type": {
        "choice": 29.411764705882355,
        "noul": -100.0,
        "score": -47.44917203444254
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0.0
      },
      "jaggedness": {
       "accuracy": 0.2918918918918919,
       "cc": -18.904988531894233,
       "cc_by_type": {
        "choice": 11.069939786938404,
        "noul": -64.1025641025641,
        "score": -3.682341280057
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.3446969696969697,
       "cc": 1.1432998727522918,
       "cc_by_type": {
        "choice": 15.466790524849063,
        "noul": -28.767123287671236,
        "score": 16.73023238107905
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 1.1432998727522918
      },
      "long_state": {
       "accuracy": 0.27586206896551724,
       "cc": -25.065654575351683,
       "cc_by_type": {
        "choice": 16.66666666666667,
        "noul": -100.0,
        "score": 8.136369607278283
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.22580645161290322,
       "cc": -33.79734650736005,
       "cc_by_type": {
        "choice": 7.692307692307688,
        "noul": -100.0,
        "score": -9.084347214387822
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.2,
       "cc": -31.03237123270632,
       "cc_by_type": {
        "choice": -1.2091356918943095,
        "noul": -93.10344827586206,
        "score": 1.2154702696374198
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.16071428571428573,
       "cc": -85.43494727513347,
       "cc_by_type": {
        "choice": -4.761904761904764,
        "noul": -90.0,
        "score": -161.54293706349563
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.2807017543859649,
       "cc": 8.99001109877914,
       "cc_by_type": {
        "choice": 8.99001109877914
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 8.99001109877914
      },
      "perturb_pair": {
       "accuracy": 0.31666666666666665,
       "cc": -11.077733022650753,
       "cc_by_type": {
        "choice": 21.36304062909567,
        "noul": -85.71428571428572,
        "score": 31.1180460172378
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 0.0
      },
      "usecase": {
       "accuracy": 0.2320675105485232,
       "cc": -25.394686423610008,
       "cc_by_type": {
        "choice": 5.184043168240532,
        "noul": -57.89473684210527,
        "score": -23.47336559696529
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.09090909090909091,
       "cc": -53.76070940660107,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -27.948794886469862
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.0,
       "cc": -63.559322033898304,
       "cc_by_type": {
        "choice": -27.11864406779661,
        "noul": -100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.375,
       "cc": -22.128997159813196,
       "cc_by_type": {
        "choice": 17.159763313609464,
        "noul": -100.0,
        "score": 16.45324520695096
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.26666666666666666,
       "cc": -26.699211006624235,
       "cc_by_type": {
        "choice": 7.6644538322269185,
        "noul": -77.77777777777779,
        "score": -9.984309074321839
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0.0
      },
      "el": {
       "accuracy": 0.16666666666666666,
       "cc": -44.641658892653076,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100.0,
        "score": -25.491241738200188
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.26273458445040215,
       "cc": -24.234183293883643,
       "cc_by_type": {
        "choice": 8.693213684800824,
        "noul": -62.40601503759399,
        "score": -18.989748528857753
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.21428571428571427,
       "cc": -23.875465734890014,
       "cc_by_type": {
        "choice": 8.363636363636363,
        "noul": -100.0,
        "score": 20.00996643169359
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.2,
       "cc": -32.05973275794293,
       "cc_by_type": {
        "choice": -1.5514809590973198,
        "noul": -100.0,
        "score": 5.372282685268514
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.2,
       "cc": -27.81219448998868,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": -100.0,
        "score": 3.519938269164402
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -48.387096774193544,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.23529411764705882,
       "cc": -47.53372129710758,
       "cc_by_type": {
        "choice": 23.83638928067701,
        "noul": -100.0,
        "score": -66.43755317199978
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      },
      "ja": {
       "accuracy": 0.125,
       "cc": -47.14183058522813,
       "cc_by_type": {
        "choice": -13.821138211382111,
        "noul": -100.0,
        "score": -27.604353544302263
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.18181818181818182,
       "cc": -43.344897587116954,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100.0,
        "score": 1.1128482222557023
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.45454545454545453,
       "cc": 16.030320645251987,
       "cc_by_type": {
        "choice": 56.04395604395604,
        "noul": -50.0,
        "score": 42.04700589179993
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 16.030320645251987
      },
      "nl": {
       "accuracy": 0.23076923076923078,
       "cc": -20.35052023493665,
       "cc_by_type": {
        "choice": 24.79108635097493,
        "noul": -66.66666666666667,
        "score": -19.17598038911821
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.14285714285714285,
       "cc": -49.18032786885246,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.225,
       "cc": -12.618486802534166,
       "cc_by_type": {
        "choice": 5.553126366418887,
        "noul": -60.0,
        "score": 16.591413225978613
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.2222222222222222,
       "cc": -28.382395955071996,
       "cc_by_type": {
        "choice": 9.547738693467334,
        "noul": -66.66666666666667,
        "score": -28.02825989201665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 0.0
      },
      "sv": {
       "accuracy": 0.0,
       "cc": -48.522135284211906,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -12.233072519302368
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.36363636363636365,
       "cc": -18.188721011256614,
       "cc_by_type": {
        "choice": -5.26315789473684,
        "noul": -100.0,
        "score": 50.696994860967
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.09090909090909091,
       "cc": -41.07722171240693,
       "cc_by_type": {
        "choice": -30.43478260869565,
        "noul": -100.0,
        "score": 7.2031174714748625
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.375,
       "cc": -10.272994610640382,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": -100.0,
        "score": 34.045881032943726
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.2853333333333333,
       "cc": 7.410606322335463,
       "cc_by_type": {
        "choice": 7.410606322335463
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 7.410606322335463
      },
      "noul": {
       "accuracy": 0.14933333333333335,
       "cc": -70.13333333333334,
       "cc_by_type": {
        "noul": -70.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.2853333333333333,
       "cc": -14.348760038030782,
       "cc_by_type": {
        "score": -14.348760038030782
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2916666666666667,
       "cc": -18.14065897262343,
       "cc_by_type": {
        "choice": -8.731466227347617,
        "noul": -19.999999999999996,
        "score": -25.690510690522682
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.16666666666666666,
       "cc": -19.845708892643234,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": -60.0,
        "score": 8.482513256603841
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.2,
       "cc": -36.093708917165806,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -75.0,
        "score": -36.506933203110336
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.19230769230769232,
       "cc": -43.411243099056556,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100.0,
        "score": -45.89637989957931
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2222222222222222,
       "cc": -19.47483978171899,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -60.0,
        "score": -5.256817481802933
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.10344827586206896,
       "cc": -52.342876150533066,
       "cc_by_type": {
        "choice": -11.111111111111114,
        "noul": -100.0,
        "score": -45.917517340488075
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.38461538461538464,
       "cc": 0.6176093078177957,
       "cc_by_type": {
        "choice": 8.695652173913043,
        "noul": 0.0,
        "score": -6.842824250459656
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0.6176093078177957
      },
      "gaming": {
       "accuracy": 0.20689655172413793,
       "cc": -26.6661863971574,
       "cc_by_type": {
        "choice": 16.831683168316832,
        "noul": -71.42857142857143,
        "score": -25.40167093121759
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": 4.412051451900017,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -100.0,
        "score": 92.60123372077942
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 4.412051451900017
      },
      "knowledge_graphs": {
       "accuracy": 0.08695652173913043,
       "cc": -55.71206240218595,
       "cc_by_type": {
        "choice": -9.923664122137408,
        "noul": -100.0,
        "score": -57.21252308442044
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.28,
       "cc": -16.228836402216572,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -100.0,
        "score": 32.61430380148036
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.23076923076923078,
       "cc": -29.703887276787327,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": 9.248993907342928
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.3103448275862069,
       "cc": -15.934116861404396,
       "cc_by_type": {
        "choice": 17.431192660550458,
        "noul": -50.0,
        "score": -15.233543244763649
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.4,
       "cc": 4.006137375452837,
       "cc_by_type": {
        "choice": 27.633342267488622,
        "noul": -40.90909090909091,
        "score": 25.294160767960793
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 4.006137375452837
      },
      "moderation_trust_safety": {
       "accuracy": 0.25,
       "cc": -29.448869234338442,
       "cc_by_type": {
        "choice": 5.263157894736842,
        "noul": -100.0,
        "score": 6.390234402247841
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.23809523809523808,
       "cc": -26.169667953406684,
       "cc_by_type": {
        "choice": 24.62311557788945,
        "noul": -100.0,
        "score": -3.132119438109493
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.23076923076923078,
       "cc": -41.00485732486038,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -100.0,
        "score": -28.67494933307171
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.15625,
       "cc": -22.981094893304945,
       "cc_by_type": {
        "choice": -11.553784860557771,
        "noul": -71.42857142857143,
        "score": 14.03907160921436
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.13636363636363635,
       "cc": -52.2527400518963,
       "cc_by_type": {
        "choice": -14.558472553699291,
        "noul": -100.0,
        "score": -42.199747601989635
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      56.7149821052611,
      66.00061446604302
     ],
     "I": [
      1.7436767620361024,
      6.437188407255578
     ],
     "capability": [
      29.712243529714982,
      35.427303807623055
     ],
     "composite": {
      "A": [
       0.007928751847345473,
       0.33896544733805034
      ],
      "B": [
       0.005122646508709089,
       0.23614958881345818
      ],
      "C": [
       0.005506077671767688,
       0.23539267176253492
      ]
     },
     "seed": 25
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.32062493293682,
     "capability": 32.79110507364784,
     "intelligence": 3.2615852143588597
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2853333333333333,
      "calibration": 54.23981005825647,
      "cc": 7.410606322335463,
      "cc_by_type": {
       "choice": 7.410606322335463
      },
      "cc_tiered": 9.78475564307658,
      "cc_tiered_open": 12.87966201889643,
      "cc_tiered_sealed": 6.689849267256729,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 7.410606322335463
     },
     "noul": {
      "accuracy": 0.14933333333333335,
      "calibration": 69.32868967393878,
      "cc": -70.13333333333334,
      "cc_by_type": {
       "noul": -70.13333333333334
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2853333333333333,
      "calibration": 63.39337506661522,
      "cc": -14.348760038030782,
      "cc_by_type": {
       "score": -14.348760038030782
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 60.8515976034804,
     "capability": 31.49910396998157,
     "intelligence": 2.1466103364827385
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.03857613577512699,
    "Emphasis on Accuracy 60:20:20": 0.022535224640971675,
    "Emphasis on Cost 20:20:60": 0.05938403884460177,
    "Emphasis on Speed 20:60:20": 0.06040919381107833,
    "Intelligence only": 0.013878616661020313
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ksonny4 / @kubelpe (ksonny4@gmail.com)",
   "axes": {
    "calibration": 66.68251917899094,
    "cost": 70.36992715953406,
    "intelligence": 2.6107469083924393,
    "speed": 57.79924670989358
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16197772093040966,
      "ece_score": 67.60445581391808,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 67.60445581391808
     },
     "noul": {
      "abstention_rate": 1,
      "brier": 0.28875559863871086,
      "ece": 0.20363870372723708,
      "n": 375,
      "score": 59.27225925455259
     },
     "score": {
      "interval90_coverage": 0.96,
      "interval90_width": 0.8991111111111114,
      "n": 375,
      "nrps": 0.1846962110408624,
      "score": 73.17084246850217,
      "top_ece": 0.17594346979454697
     }
    },
    "score": 66.68251917899094
   },
   "capability": 34.64663304369169,
   "composite_ci95": {
    "A": [
     0.0008967609092971513,
     0.12984410368517996
    ],
    "B": [
     0.0005711694021380616,
     0.08896822577663006
    ],
    "C": [
     0.000622750631456355,
     0.09016951644804166
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public tariff. The same choice the published quyet-1-0-small-en row carries for the identical ModernBERT-base backbone: DeepInfra base-size encoders at USD 0.005 per 1M input and 0 output (deepinfra:thenlper/gte-base, BAAI/bge-base-en-v1.5, intfloat/e5-base-v2 and sentence-transformers/all-mpnet-base-v2 are all 5e-7 cents per input token in the frozen 25 Sep 2026 DeepInfra snapshot). Output tokens are a literal 0 reported by the author's server (NLI entailment read-out, nothing is generated). This row's own usage on draw v1.6-regular-20261010-a2: 2,916,019 input and 0 output tokens over 1,500 decisions. Estimated, not charged.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009720063333333334
   },
   "decision_limit": null,
   "display": "modernbert-jev v2",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s. The author's own CLI at github.com/KSonny4/modernbert-jev@b1866e4d, `modernbert-jev serve --model <pinned snapshot dir>` at its own defaults (1 ONNX Runtime thread, 127.0.0.1), inside a loopback-only network namespace with an outbound probe that failed as required, no credentials in the environment. int8 (unsigned) ONNX as shipped. Speed is a lower bound for this model on a dedicated core.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 3.8269990036680306,
    "I_sealed": 1.394494813116848,
    "base": 2.6107469083924393,
    "excess": -7.6407403907885305,
    "gap": 2.4325041905511826,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 11.480997011004092,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486504,
       "hard": -1.9188424032714584,
       "judge": 20.89650626235992,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 4.480029631602289,
       "hard": -0.07960342036494872,
       "judge": 1.7876323239060077,
       "standard": -15.095565734194793
      }
     },
     "sealed|choice": {
      "cc": 4.183484439350544,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 8.019605798310561,
       "hard": 2.055622732769036,
       "judge": 3.2012581807112888,
       "standard": 7.994486560992435
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 1.478176928885322,
       "hard": -3.5620793106463378,
       "judge": -5.974082989668261,
       "standard": -11.414183075878537
      }
     }
    }
   },
   "jevbench_score": 0.025388991293562355,
   "key": "modernbert-jev-v2",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (model card; the code README and pyproject.toml also declare Apache-2.0, without a separate LICENSE file)",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.8",
   "measurement_date_status": "measured",
   "method": "v1.6",
   "model_pin": "ksonny4/modernbert-jev-v2@9708136f22a85b29fcf90d3960aa642d9eadcb1c",
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive": 0,
    "decisive_rate": 0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 66.68251917899094,
     "cost": 70.36992715953406,
     "intelligence": 2.6107469083924393,
     "speed": 57.79924670989358
    },
    "board_release": "v1.6.43",
    "capability": 34.64663304369169,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.0008967609092971513,
      0.12984410368517996
     ],
     "B": [
      0.0005711694021380616,
      0.08896822577663006
     ],
     "C": [
      0.000622750631456355,
      0.09016951644804166
     ]
    },
    "jevbench_score": 0.025388991293562355,
    "measured_in": "v1.6.8",
    "method": "v1.6",
    "open_board_rank": 185,
    "rank": 229,
    "ranks": {
     "A": 229,
     "B": 229,
     "C": 229,
     "capability": 217
    },
    "scores": {
     "A": 0.025388991293562355,
     "B": 0.016776345440377434,
     "C": 0.01763124395386275
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 5.349294321238995,
     "p50_s_raw": 2.5996471606194973,
     "p95_s_adjusted": 31.02978992164135,
     "p95_s_raw": 15.439894960820675
    },
    "views": {
     "Balanced 33:33:33": 0.01973059249456439,
     "Emphasis on Accuracy 60:20:20": 0.011546588753666046,
     "Emphasis on Cost 20:20:60": 0.030774419703851856,
     "Emphasis on Speed 20:60:20": 0.030350892509696595,
     "Intelligence only": 0.0071179397650143975
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/ksonny4/modernbert-jev-v2",
   "scores": {
    "A": 0.025388991293562355,
    "B": 0.016776345440377434,
    "C": 0.01763124395386275
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 5.349294321238995,
    "p50_s_raw": 2.5996471606194973,
    "p95_s_adjusted": 31.02978992164135,
    "p95_s_raw": 15.439894960820675
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.13793103448275862,
       "cc": -43.39863396756374,
       "cc_by_type": {
        "choice": -8.333333333333334,
        "noul": -100,
        "score": -21.862568569357887
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.17857142857142858,
       "cc": -45.8159894104131,
       "cc_by_type": {
        "choice": 2.173913043478264,
        "noul": -100,
        "score": -39.621881274717595
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.22448979591836735,
       "cc": -32.536005477410086,
       "cc_by_type": {
        "choice": 3.7037037037037055,
        "noul": -100,
        "score": -1.3117201359339692
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3333333333333333,
       "cc": -25.18773661803903,
       "cc_by_type": {
        "choice": -5.882352941176475,
        "noul": -100,
        "score": 30.319143087059388
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.14594594594594595,
       "cc": -46.20038451819038,
       "cc_by_type": {
        "choice": -12.552107457156087,
        "noul": -100,
        "score": -26.04904609741505
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.18181818181818182,
       "cc": -31.90735375118363,
       "cc_by_type": {
        "choice": 10.822108685555056,
        "noul": -100,
        "score": -6.544169939105937
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0.034482758620689655,
       "cc": -44.327454537857705,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -2.0299826611921645
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.1935483870967742,
       "cc": -36.52576519511609,
       "cc_by_type": {
        "choice": -1.5384615384615408,
        "noul": -100,
        "score": -8.038834046886723
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.20952380952380953,
       "cc": -28.34675764556054,
       "cc_by_type": {
        "choice": 6.8517689207344405,
        "noul": -100,
        "score": 8.107958142583938
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.14285714285714285,
       "cc": -56.58208630213584,
       "cc_by_type": {
        "choice": -4.761904761904764,
        "noul": -100,
        "score": -64.98435414450276
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.21052631578947367,
       "cc": 0.11098779134295962,
       "cc_by_type": {
        "choice": 0.11098779134295962
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 0.11098779134295962
      },
      "perturb_pair": {
       "accuracy": 0.2833333333333333,
       "cc": -18.717250201930284,
       "cc_by_type": {
        "choice": 17.431192660550455,
        "noul": -100,
        "score": 26.417056733658693
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.2489451476793249,
       "cc": -25.635511113946517,
       "cc_by_type": {
        "choice": 21.37213335902873,
        "noul": -100,
        "score": 1.7213332991317332
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.2727272727272727,
       "cc": -34.51061721771425,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": -100,
        "score": 3.134815013523906
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.125,
       "cc": -50.847457627118644,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.3333333333333333,
       "cc": -20.541065093618435,
       "cc_by_type": {
        "choice": 40.8284023668639,
        "noul": -100,
        "score": -2.451597647719206
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.2,
       "cc": -32.195675052815936,
       "cc_by_type": {
        "choice": 2.232951116475559,
        "noul": -100,
        "score": 1.1800237250766288
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.16666666666666666,
       "cc": -50.44316644371644,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100,
        "score": -54.94395716247462
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.19571045576407506,
       "cc": -34.73453804072174,
       "cc_by_type": {
        "choice": 4.206393718451972,
        "noul": -100,
        "score": -8.410007840617183
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.25,
       "cc": -18.257637042739912,
       "cc_by_type": {
        "choice": 21.45454545454546,
        "noul": -100,
        "score": 23.772543417234793
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.4,
       "cc": -12.310038757433736,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -100,
        "score": -11.542246032526894
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.13333333333333333,
       "cc": -35.49650623401251,
       "cc_by_type": {
        "choice": -11.801242236024846,
        "noul": -100,
        "score": 5.3117235339873226
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.3333333333333333,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.29411764705882354,
       "cc": -22.98844453863286,
       "cc_by_type": {
        "choice": 23.83638928067701,
        "noul": -100,
        "score": 7.198277103424422
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.125,
       "cc": -40.139871804621805,
       "cc_by_type": {
        "choice": -13.821138211382111,
        "noul": -100,
        "score": -6.598477202483299
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.2727272727272727,
       "cc": -36.975867376244004,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": -12.56694639102709
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0,
       "cc": -40.20521661272176,
       "cc_by_type": {
        "choice": -31.868131868131872,
        "noul": -100,
        "score": 11.252482029966604
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.07692307692307693,
       "cc": -29.998198543905005,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": -100,
        "score": 10.283955900318398
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.2857142857142857,
       "cc": -32.78688524590164,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.2,
       "cc": -24.81588086706398,
       "cc_by_type": {
        "choice": 5.553126366418887,
        "noul": -100,
        "score": 19.999231032389165
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.2777777777777778,
       "cc": -9.14274565490507,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": -100,
        "score": 8.752667557897864
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0,
       "cc": -43.590934068488735,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": 2.5605311278671383
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.36363636363636365,
       "cc": -17.28662918547593,
       "cc_by_type": {
        "choice": 21.052631578947373,
        "noul": -100,
        "score": 27.087480864624837
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.2727272727272727,
       "cc": -23.219083224832847,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100,
        "score": 39.038402499414495
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0,
       "cc": -44.80276763421367,
       "cc_by_type": {
        "choice": -29.729729729729726,
        "noul": -100,
        "score": -4.678573172911271
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.268,
       "cc": 5.164968042839871,
       "cc_by_type": {
        "choice": 5.164968042839871
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 5.164968042839871
      },
      "noul": {
       "accuracy": 0,
       "cc": -100,
       "cc_by_type": {
        "noul": -100
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.2613333333333333,
       "cc": -4.528144639423148,
       "cc_by_type": {
        "score": -4.528144639423148
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.375,
       "cc": -20.45434194583145,
       "cc_by_type": {
        "choice": 30.807248764415156,
        "noul": -100,
        "score": 7.829725398090492
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -23.589501736030243,
       "cc_by_type": {
        "choice": 31.26022913256956,
        "noul": -100,
        "score": -2.0287343406602876
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.15,
       "cc": -30.19926598996379,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": 6.176395578495719
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.15384615384615385,
       "cc": -40.108407064559685,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100,
        "score": -11.891486253920025
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -17.392819381831114,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": 3.722162972519083
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.1724137931034483,
       "cc": -35.42851137104405,
       "cc_by_type": {
        "choice": -2.564102564102566,
        "noul": -100,
        "score": -3.721431549029597
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.19230769230769232,
       "cc": -26.23012292511098,
       "cc_by_type": {
        "choice": 8.695652173913043,
        "noul": -100,
        "score": 12.61397905075401
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.1724137931034483,
       "cc": -30.62771796722504,
       "cc_by_type": {
        "choice": 4.950495049504946,
        "noul": -100,
        "score": 3.1663510488199376
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.42857142857142855,
       "cc": 9.59819138604467,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -100,
        "score": 76.41362177718163
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 9.59819138604467
      },
      "knowledge_graphs": {
       "accuracy": 0.13043478260869565,
       "cc": -38.28700352328199,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100,
        "score": -14.09765179122
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.16,
       "cc": -32.661969492218304,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -100,
        "score": -0.4249328668988195
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.38461538461538464,
       "cc": -15.740475832931393,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 18.352342993009096
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.3448275862068966,
       "cc": -22.60830157830547,
       "cc_by_type": {
        "choice": 26.60550458715596,
        "noul": -100,
        "score": 5.569590677927638
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.21333333333333335,
       "cc": -26.190070048541028,
       "cc_by_type": {
        "choice": 19.592602519431804,
        "noul": -100,
        "score": 1.8371873349451118
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -3.248277111101165,
       "cc_by_type": {
        "choice": 68.42105263157895,
        "noul": -100,
        "score": 21.834116035117557
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.19047619047619047,
       "cc": -28.752891240849554,
       "cc_by_type": {
        "choice": 9.547738693467341,
        "noul": -100,
        "score": 4.193587583984004
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.15384615384615385,
       "cc": -37.98670463078509,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -100,
        "score": -19.620491250845838
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.15625,
       "cc": -32.51983448550218,
       "cc_by_type": {
        "choice": -3.585657370517929,
        "noul": -100,
        "score": 6.0261539140113785
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.09090909090909091,
       "cc": -35.388151349291284,
       "cc_by_type": {
        "choice": -14.558472553699291,
        "noul": -100,
        "score": 8.394018505825429
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      61.644678187031296,
      70.74968953890287
     ],
     "I": [
      0.8350318363608307,
      4.617476925221922
     ],
     "capability": [
      31.94442581392808,
      37.14219934722527
     ],
     "composite": {
      "A": [
       0.0008967609092971513,
       0.12984410368517996
      ],
      "B": [
       0.0005711694021380616,
       0.08896822577663006
      ],
      "C": [
       0.000622750631456355,
       0.09016951644804166
      ]
     },
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.68251917899094,
     "capability": 34.64663304369169,
     "intelligence": 2.6107469083924393
    },
    "per_type": {
     "choice": {
      "accuracy": 0.268,
      "calibration": 67.60445581391808,
      "cc": 5.164968042839871,
      "cc_by_type": {
       "choice": 5.164968042839871
      },
      "cc_tiered": 7.832240725177318,
      "cc_tiered_open": 11.480997011004092,
      "cc_tiered_sealed": 4.183484439350544,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 5.164968042839871
     },
     "noul": {
      "accuracy": 0,
      "calibration": 59.27225925455259,
      "cc": -100,
      "cc_by_type": {
       "noul": -100
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2613333333333333,
      "calibration": 73.17084246850217,
      "cc": -4.528144639423148,
      "cc_by_type": {
       "score": -4.528144639423148
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 59.34147162065773,
     "capability": 31.140405932959595,
     "intelligence": 2.939340245261466
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.01973059249456439,
    "Emphasis on Accuracy 60:20:20": 0.011546588753666046,
    "Emphasis on Cost 20:20:60": 0.030774419703851856,
    "Emphasis on Speed 20:60:20": 0.030350892509696595,
    "Intelligence only": 0.0071179397650143975
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Shanghua Gao (@GaoShanghua)",
   "axes": {
    "calibration": 85.34979344070516,
    "cost": 31.141652902933714,
    "intelligence": 61.564156247760415,
    "speed": 84.6270713604755
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08475596558784464,
      "ece_score": 83.04880688243107,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 83.04880688243107,
      "score_before_time_limit": 83.04880688243107,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.17307692307692313,
      "brier": 0.10655681075571934,
      "ece": 0.06711648937487673,
      "n": 364,
      "score": 86.57670212502465,
      "score_before_time_limit": 86.57670212502465,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9402173913043478,
      "interval90_width": 0.263586956521739,
      "n": 368,
      "nrps": 0.07024138314041055,
      "score": 86.42387131465975,
      "score_before_time_limit": 86.42387131465975,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10064059528319728
     }
    },
    "score": 85.34979344070516
   },
   "capability": 73.45697484423279,
   "composite_ci95": {
    "A": [
     21.059394581478518,
     21.881943653496982
    ],
    "B": [
     21.236351182193502,
     22.477670325097492
    ],
    "C": [
     18.816668135960498,
     21.881943653496982
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference. The declared base Qwen/Qwen3.8-27B resolves in the frozen 25 Sep 2026 snapshot through pricing_v15 BASE_REFERENCES to qwen/qwen3.8-27b at USD 0.42 per 1M input and USD 3.00 per 1M output (verified before the pod was rented; r75 COST-BASIS.json). This row reports a real output token count - 1 token per readout, 1,471 in total - so the Gutsy zero-output rule does NOT apply and the output price is charged as measured. This row's own usage on draw v1.6-regular-20261010-a2: 694,381 input and 1,471 output tokens over 1,500 decisions.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.19736867999999996
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RSI-Jev v6.1-VL 27B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium H100 80GB pod (36 min, USD 1.95/h), sealed per chief-of-staff decision #16415 on Florian's YES: the author's own serve.cli inside a loopback-only network namespace (unshare -n; DNS and outbound TCP both proven dead from inside the run namespace), started with env -i and OPENAI_API_KEY empty, plus a pod-level iptables egress cut installed and verified before the sealed input was uploaded. HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE set only to block the download fallbacks the r75 review found; no source line was edited. Server pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, not a repo id, so no revision could resolve to main. Unchanged defaults, --effort auto, 32,768-token cap, no truncation. 54,047 MiB of 81,559 MiB used.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 68.48613704048307,
    "I_sealed": 54.64217545503776,
    "base": 61.564156247760415,
    "excess": 3.7707170041056006,
    "gap": 13.843961585445314,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.8256733782429,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 73.57659641396666,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 66.15458937198068,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 63.478148371225664,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.937072018553,
       "hard": 66.44443112298649,
       "judge": 61.06082661163181,
       "standard": 58.942103683431114
      }
     },
     "sealed|choice": {
      "cc": 70.33653729047333,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 88.73709458754823,
       "hard": 64.87853138397273,
       "judge": 71.69093399624576,
       "standard": 70.02067539627843
      }
     },
     "sealed|noul": {
      "cc": 41.66298613667034,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 58.974358974358964,
       "hard": 21.212121212121215,
       "judge": 50.0,
       "standard": 61.403508771929815
      }
     },
     "sealed|score": {
      "cc": 51.92700293796962,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 68.04656098024147,
       "hard": 33.730600016105896,
       "judge": 64.21257035928524,
       "standard": 61.83167862858772
      }
     }
    }
   },
   "jevbench_score": 21.584866751250143,
   "key": "rsi-jev-v6.1-vl-27b",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (model card); source Shanghua-Gao/RSI-Jev@fc982bbd MIT",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "shgao/rsi-jev-v6.1-vl-27b@d0dccc052ee844ab7481ddfacad1b7335ab394e0",
   "noul_decisive": {
    "acc_among_decisive": 0.9102990033222591,
    "decisive": 301,
    "decisive_rate": 0.8269230769230769,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.34979344070516,
     "cost": 31.141652902933714,
     "intelligence": 61.564156247760415,
     "speed": 84.71397705257328
    },
    "board_release": "v1.6.43",
    "capability": 73.45697484423279,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      21.062859904802277,
      21.88568498833731
     ],
     "B": [
      21.239170136076478,
      22.480828485144727
     ],
     "C": [
      18.819765630154432,
      21.88568498833731
     ]
    },
    "jevbench_score": 21.58850717988564,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 87,
    "rank": 120,
    "ranks": {
     "A": 120,
     "B": 116,
     "C": 102,
     "capability": 56
    },
    "scores": {
     "A": 21.58850717988564,
     "B": 22.011280090800952,
     "C": 21.58850717988564
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.39549588703666816,
     "p50_s_raw": 0.12274794351833407,
     "p95_s_adjusted": 0.8540048229158855,
     "p95_s_raw": 0.35200241145794275
    },
    "views": {
     "Balanced 33:33:33": 19.344794057306224,
     "Emphasis on Accuracy 60:20:20": 20.935791627644683,
     "Emphasis on Cost 20:20:60": 15.593981583505036,
     "Emphasis on Speed 20:60:20": 23.15453324264071,
     "Intelligence only": 23.882030176915077
    }
   },
   "rank": 136,
   "ranked": true,
   "ranks": {
    "A": 136,
    "B": 131,
    "C": 117,
    "capability": 66
   },
   "repo": "https://huggingface.co/shgao/rsi-jev-v6.1-vl-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.584866751250143,
    "B": 22.008252470671053,
    "C": 21.584866751250143
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4150659809878562,
    "p50_s_raw": 0.13253299049392808,
    "p95_s_adjusted": 0.830186609426164,
    "p95_s_raw": 0.34009330471308197
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9310344827586207,
       "cc": 86.4125424016998,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 92.5709605384327
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 86.4125424016998
      },
      "calibration_gen": {
       "accuracy": 0.7232142857142857,
       "cc": 58.19506668349169,
       "cc_by_type": {
        "choice": 67.3913043478261,
        "noul": 61.53846153846154,
        "score": 45.655434164187426
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 58.19506668349169
      },
      "composite_gen": {
       "accuracy": 0.8877551020408163,
       "cc": 82.46075900003056,
       "cc_by_type": {
        "choice": 96.29629629629629,
        "noul": 78.57142857142858,
        "score": 72.51455213236682
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 82.46075900003056
      },
      "fairness": {
       "accuracy": 0.9629629629629629,
       "cc": 94.1963028792578,
       "cc_by_type": {
        "choice": 88.23529411764707,
        "noul": 100.0,
        "score": 94.35361452012633
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 94.1963028792578
      },
      "jaggedness": {
       "accuracy": 0.5081081081081081,
       "cc": 13.570657148702269,
       "cc_by_type": {
        "choice": 38.86058360352014,
        "noul": -48.71794871794872,
        "score": 50.56933656053539
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 13.570657148702269
      },
      "llm_router": {
       "accuracy": 0.8598484848484849,
       "cc": 81.63698812259595,
       "cc_by_type": {
        "choice": 75.84765443567116,
        "noul": 86.30136986301369,
        "score": 82.76194006910303
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 81.63698812259595
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6774193548387096,
       "cc": 31.38414817065086,
       "cc_by_type": {
        "choice": 72.30769230769232,
        "noul": 0.0,
        "score": 21.844752204260253
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 31.38414817065086
      },
      "multilingual": {
       "accuracy": 0.7428571428571429,
       "cc": 58.6257780124704,
       "cc_by_type": {
        "choice": 70.44334975369458,
        "noul": 51.72413793103448,
        "score": 53.70984635268214
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 58.6257780124704
      },
      "negation_pair_gen": {
       "accuracy": 0.875,
       "cc": 63.20741226657981,
       "cc_by_type": {
        "choice": 90.47619047619048,
        "noul": 60.00000000000001,
        "score": 39.146046323548944
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 63.20741226657981
      },
      "option_order": {
       "accuracy": 0.9824561403508771,
       "cc": 97.78024417314094,
       "cc_by_type": {
        "choice": 97.78024417314094
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 97.78024417314094
      },
      "perturb_pair": {
       "accuracy": 0.9,
       "cc": 89.49429032954704,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 100.0,
        "score": 84.21026286282202
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 89.49429032954704
      },
      "usecase": {
       "accuracy": 0.7046413502109705,
       "cc": 51.83295324947296,
       "cc_by_type": {
        "choice": 73.40528040084796,
        "noul": 36.8421052631579,
        "score": 45.251474084413026
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 51.83295324947296
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8181818181818182,
       "cc": 69.11348259490397,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 50.0,
        "score": 84.00711445137858
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 69.11348259490397
      },
      "cs": {
       "accuracy": 0.875,
       "cc": 87.28813559322035,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 87.28813559322035
      },
      "da": {
       "accuracy": 0.6666666666666666,
       "cc": 44.00477315138941,
       "cc_by_type": {
        "choice": 52.662721893491124,
        "noul": 19.999999999999996,
        "score": 59.351597560677114
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 44.00477315138941
      },
      "de": {
       "accuracy": 0.8444444444444444,
       "cc": 72.3799653014141,
       "cc_by_type": {
        "choice": 83.70549185274592,
        "noul": 77.77777777777777,
        "score": 55.656626273718615
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 72.3799653014141
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": -0.8800661948371129,
       "cc_by_type": {
        "choice": 75.90361445783132,
        "noul": -19.999999999999996,
        "score": -58.54381304234266
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.7497765862377123,
       "cc": 57.06152962458927,
       "cc_by_type": {
        "choice": 72.18171620863711,
        "noul": 42.85714285714286,
        "score": 56.14572980798784
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 57.06152962458927
      },
      "es": {
       "accuracy": 0.8928571428571429,
       "cc": 82.61244146186118,
       "cc_by_type": {
        "choice": 86.90909090909093,
        "noul": 100.0,
        "score": 60.9282334764926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 82.61244146186118
      },
      "fi": {
       "accuracy": 0.5555555555555556,
       "cc": 34.97652582159624,
       "cc_by_type": {
        "choice": 36.61971830985915,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 34.97652582159624
      },
      "fr": {
       "accuracy": 0.6666666666666666,
       "cc": 41.96733456321797,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 0.0,
        "score": 51.28987392942821
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 41.96733456321797
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 52.85728930896761,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 60.00000000000001,
        "score": 35.83894867224444
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 52.85728930896761
      },
      "id": {
       "accuracy": 0.8333333333333334,
       "cc": 83.87096774193549,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 83.87096774193549
      },
      "it": {
       "accuracy": 1.0,
       "cc": 90.29663262626855,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 70.88989787880564
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 90.29663262626855
      },
      "ja": {
       "accuracy": 0.53125,
       "cc": 32.05622061795162,
       "cc_by_type": {
        "choice": 43.08943089430895,
        "noul": 14.28571428571428,
        "score": 38.79351667383163
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 32.05622061795162
      },
      "ko": {
       "accuracy": 0.8181818181818182,
       "cc": 85.45781546660389,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 56.373446399811655
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 85.45781546660389
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.9689563002903,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.9068689008709
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 99.9689563002903
      },
      "nl": {
       "accuracy": 0.6923076923076923,
       "cc": 40.991198714143415,
       "cc_by_type": {
        "choice": 74.93036211699165,
        "noul": 33.33333333333333,
        "score": 14.709900692105283
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 40.991198714143415
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.7,
       "cc": 51.32398893140217,
       "cc_by_type": {
        "choice": 92.12942719720158,
        "noul": 19.999999999999996,
        "score": 41.84253959700492
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 51.32398893140217
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 83.10065347581066,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": 100.0,
        "score": 85.48286495004504
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 83.10065347581066
      },
      "sv": {
       "accuracy": 0.625,
       "cc": 53.29148096066934,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50.0,
        "score": 98.76333177089691
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 53.29148096066934
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 23.85319314395998,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -35.457964427769184
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 23.85319314395998
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": 4.1198605058986955,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100.0,
        "score": 77.5769728220439
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 4.1198605058986955
      },
      "zh": {
       "accuracy": 0.875,
       "cc": 73.56783818557031,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 33.33333333333333,
        "score": 87.37018122337759
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 73.56783818557031
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7826666666666666,
       "cc": 71.84315080324754,
       "cc_by_type": {
        "choice": 71.84315080324754
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.84315080324754
      },
      "noul": {
       "accuracy": 0.7306666666666667,
       "cc": 46.13333333333334,
       "cc_by_type": {
        "noul": 46.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 46.13333333333334
      },
      "score": {
       "accuracy": 0.6986666666666667,
       "cc": 54.531190100580204,
       "cc_by_type": {
        "score": 54.531190100580204
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 54.531190100580204
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7083333333333334,
       "cc": 36.966773340637815,
       "cc_by_type": {
        "choice": 70.34596375617794,
        "noul": -19.999999999999996,
        "score": 60.55435626573551
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 36.966773340637815
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 68.95860502475776,
       "cc_by_type": {
        "choice": 70.54009819967267,
        "noul": 100.0,
        "score": 36.33571687460061
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 68.95860502475776
      },
      "customer_support": {
       "accuracy": 0.75,
       "cc": 47.499513085946944,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 75.0,
        "score": -0.24339622603013478
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 47.499513085946944
      },
      "demand_forecasting": {
       "accuracy": 0.3076923076923077,
       "cc": -17.782585557336656,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -50.0,
        "score": -19.010407274419606
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5555555555555556,
       "cc": 28.57990326207394,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -19.999999999999996,
        "score": 43.006790531563425
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 28.57990326207394
      },
      "feature_extraction": {
       "accuracy": 0.8620689655172413,
       "cc": 64.36666498139438,
       "cc_by_type": {
        "choice": 82.90598290598291,
        "noul": 19.999999999999996,
        "score": 90.19401203820024
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 64.36666498139438
      },
      "financial_crime": {
       "accuracy": 0.7692307692307693,
       "cc": 69.24951206795548,
       "cc_by_type": {
        "choice": 86.95652173913045,
        "noul": 100.0,
        "score": 20.792014464735985
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 69.24951206795548
      },
      "gaming": {
       "accuracy": 0.5862068965517241,
       "cc": 25.37172392511331,
       "cc_by_type": {
        "choice": 52.475247524752476,
        "noul": -14.28571428571429,
        "score": 37.92563853630174
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 25.37172392511331
      },
      "insurance_claims": {
       "accuracy": 0.5714285714285714,
       "cc": 31.669815093397148,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -19.999999999999996,
        "score": 62.62849289923906
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 31.669815093397148
      },
      "knowledge_graphs": {
       "accuracy": 0.6956521739130435,
       "cc": 42.704203038079555,
       "cc_by_type": {
        "choice": 72.51908396946564,
        "noul": 19.999999999999996,
        "score": 35.59352514477303
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 42.704203038079555
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 55.47104563667543,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 25.0,
        "score": 41.413136910026275
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 55.47104563667543
      },
      "legal_compliance": {
       "accuracy": 0.6923076923076923,
       "cc": 8.840061259572431,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -100.0,
        "score": 59.30706902461893
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 8.840061259572431
      },
      "llm_guardrails": {
       "accuracy": 0.9310344827586207,
       "cc": 89.90702297249,
       "cc_by_type": {
        "choice": 90.8256880733945,
        "noul": 100.0,
        "score": 78.89538084407552
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 89.90702297249
      },
      "model_routing": {
       "accuracy": 0.86,
       "cc": 76.7397155888396,
       "cc_by_type": {
        "choice": 85.52666845349772,
        "noul": 81.81818181818181,
        "score": 62.874296494839264
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 76.7397155888396
      },
      "moderation_trust_safety": {
       "accuracy": 0.6875,
       "cc": 52.07409272558595,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 60.00000000000001,
        "score": 43.590699229389415
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 52.07409272558595
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 54.013036775459085,
       "cc_by_type": {
        "choice": 54.77386934673366,
        "noul": 66.66666666666667,
        "score": 40.59857431297692
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 54.013036775459085
      },
      "risk_assessment": {
       "accuracy": 0.6153846153846154,
       "cc": 46.84943347419826,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 33.33333333333333,
        "score": 44.95081614586523
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 46.84943347419826
      },
      "scientific_discovery": {
       "accuracy": 0.59375,
       "cc": 25.763079128747716,
       "cc_by_type": {
        "choice": 60.1593625498008,
        "noul": -14.28571428571429,
        "score": 31.41558912215664
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 25.763079128747716
      },
      "search_retrieval": {
       "accuracy": 0.9090909090909091,
       "cc": 86.99121578361938,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 60.97364735085814
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 86.99121578361938
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.64755807175129,
      87.47437139642248
     ],
     "I": [
      56.725930974421935,
      65.23484456387472
     ],
     "capability": [
      69.35974616373294,
      75.75238339213846
     ],
     "composite": {
      "A": [
       21.059394581478518,
       21.881943653496982
      ],
      "B": [
       21.236351182193502,
       22.477670325097492
      ],
      "C": [
       18.816668135960498,
       21.881943653496982
      ]
     },
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.34979344070516,
     "capability": 73.45697484423279,
     "intelligence": 61.564156247760415
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7826666666666666,
      "calibration": 83.04880688243107,
      "cc": 71.84315080324754,
      "cc_by_type": {
       "choice": 71.84315080324754
      },
      "cc_tiered": 73.08110533435811,
      "cc_tiered_open": 75.8256733782429,
      "cc_tiered_sealed": 70.33653729047333,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.84315080324754
     },
     "noul": {
      "accuracy": 0.7306666666666667,
      "calibration": 86.57670212502465,
      "cc": 46.13333333333334,
      "cc_by_type": {
       "noul": 46.13333333333334
      },
      "cc_tiered": 53.90878775432551,
      "cc_tiered_open": 66.15458937198068,
      "cc_tiered_sealed": 41.66298613667034,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.13333333333334
     },
     "score": {
      "accuracy": 0.6986666666666667,
      "calibration": 86.42387131465975,
      "cc": 54.531190100580204,
      "cc_by_type": {
       "score": 54.531190100580204
      },
      "cc_tiered": 57.70257565459764,
      "cc_tiered_open": 63.478148371225664,
      "cc_tiered_sealed": 51.92700293796962,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.531190100580204
     }
    },
    "raw_on_A": {
     "calibration": 84.95320234939237,
     "capability": 73.13630404134082,
     "intelligence": 61.31940573328927
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.340896791814014,
    "Emphasis on Accuracy 60:20:20": 20.933052624840787,
    "Emphasis on Cost 20:20:60": 15.592461938195232,
    "Emphasis on Speed 20:60:20": 23.14448534186175,
    "Intelligence only": 23.882030176915077
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking",
   "axes": {
    "calibration": 85.49831885164681,
    "cost": 43.21124696592469,
    "intelligence": 34.464905061466695,
    "speed": 84.17488059602204
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07091377459977057,
      "ece_score": 85.8172450800459,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 85.8172450800459,
      "score_before_time_limit": 85.8172450800459,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.39010989010989006,
      "brier": 0.15382831780922446,
      "ece": 0.0674352050569083,
      "n": 364,
      "score": 86.51295898861834,
      "score_before_time_limit": 86.51295898861834,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9320652173913043,
      "interval90_width": 0.5106431159420289,
      "n": 368,
      "nrps": 0.11464419484173276,
      "score": 84.1647524862762,
      "score_before_time_limit": 84.1647524862762,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10103037771637155
     }
    },
    "score": 85.49831885164681
   },
   "capability": 59.981611956556755,
   "composite_ci95": {
    "A": [
     12.630725624836908,
     23.78332080006789
    ],
    "B": [
     11.12675433052624,
     21.913417722125814
    ],
    "C": [
     8.771337239470075,
     16.516195000047144
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: model card base_model StandardThinking/StandardOne-8B, built on mistralai/Ministral-3-8B-Instruct-2512 (not listed on OpenRouter as Standard One). OpenRouter mistralai/ministral-8b-2512 list price USD 0.15/M input and USD 0.15/M output, read 2026-10-10T08:43Z from https://openrouter.ai/api/v1/models. Schema head generates no text: 0 output tokens.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0781553
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 8B SH (Standard Thinking)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline read-only container (--network none); author's reference server code/serve_head.py, unchanged defaults",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.65983609411336,
    "I_sealed": 26.269974028820027,
    "base": 34.464905061466695,
    "excess": 6.316617483953619,
    "gap": 16.389862065293332,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 64.76188322786868,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432432,
       "hard": 56.59012268008808,
       "judge": 64.40342781806197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 26.588681849551417,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 36.628943204920006,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 20.91980977072071,
       "hard": 40.258980287858556,
       "judge": 46.927298157653716,
       "standard": 21.775903327041945
      }
     },
     "sealed|choice": {
      "cc": 53.894778364676796,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 60.57983105641882,
       "hard": 50.5331427943278,
       "judge": 51.60062909035564,
       "standard": 60.716747070985534
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 12.82051282051282,
       "hard": -9.090909090909093,
       "judge": 11.111111111111116,
       "standard": -8.771929824561408
      }
     },
     "sealed|score": {
      "cc": 24.915143721783284,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 48.44464458742374,
       "hard": 10.686057111566038,
       "judge": 28.029772063789803,
       "standard": 36.93662399638777
      }
     }
    }
   },
   "jevbench_score": 18.74307320095725,
   "key": "standard_one_8b_sh",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (merged weights, schema head and reference server; base Ministral 3 8B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "StandardThinking/StandardOne-8B-SH@25e5739bd9635b54f925cf9ba2daa2ec42afd19e",
   "noul_decisive": {
    "acc_among_decisive": 0.8828828828828829,
    "decisive": 222,
    "decisive_rate": 0.6098901098901099,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.49831885164681,
     "cost": 43.21124696592469,
     "intelligence": 34.464905061466695,
     "speed": 84.28099014807135
    },
    "board_release": "v1.6.43",
    "capability": 59.981611956556755,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      12.633047380555162,
      23.788181701570505
     ],
     "B": [
      11.128195660262914,
      21.916709714788084
     ],
     "C": [
      8.772949569829972,
      16.519570626090626
     ]
    },
    "jevbench_score": 18.74677558938401,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 103,
    "rank": 138,
    "ranks": {
     "A": 138,
     "B": 141,
     "C": 144,
     "capability": 132
    },
    "scores": {
     "A": 18.74677558938401,
     "B": 16.94153091767041,
     "C": 13.018594159294452
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27213579495437445,
     "p50_s_raw": 0.061067897477187216,
     "p95_s_adjusted": 1.3712458158901424,
     "p95_s_raw": 0.6106229079450712
    },
    "views": {
     "Balanced 33:33:33": 16.62869989264526,
     "Emphasis on Accuracy 60:20:20": 14.53757347040618,
     "Emphasis on Cost 20:20:60": 16.08558260894422,
     "Emphasis on Speed 20:60:20": 20.21989737661098,
     "Intelligence only": 12.230517698051182
    }
   },
   "rank": 156,
   "ranked": true,
   "ranks": {
    "A": 156,
    "B": 159,
    "C": 162,
    "capability": 150
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-8B-SH",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.74307320095725,
    "B": 16.939111850803307,
    "C": 13.016023056220313
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28489439501427116,
    "p50_s_raw": 0.06744719750713557,
    "p95_s_adjusted": 1.3422333808033726,
    "p95_s_raw": 0.5961166904016864
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.896551724137931,
       "cc": 76.24972247673712,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 45.41583409687805
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 76.24972247673712
      },
      "calibration_gen": {
       "accuracy": 0.6339285714285714,
       "cc": 40.6073873879211,
       "cc_by_type": {
        "choice": 60.40372670807453,
        "noul": 30.76923076923077,
        "score": 30.64920468645799
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 40.6073873879211
      },
      "composite_gen": {
       "accuracy": 0.6428571428571429,
       "cc": 42.991452161771214,
       "cc_by_type": {
        "choice": 70.37037037037037,
        "noul": 28.57142857142858,
        "score": 30.032557543514706
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 42.991452161771214
      },
      "fairness": {
       "accuracy": 0.9629629629629629,
       "cc": 88.7254774240554,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 66.17643227216622
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 88.7254774240554
      },
      "jaggedness": {
       "accuracy": 0.4702702702702703,
       "cc": 8.921712067552678,
       "cc_by_type": {
        "choice": 34.69198703103288,
        "noul": -53.84615384615385,
        "score": 45.919303017779
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 8.921712067552678
      },
      "llm_router": {
       "accuracy": 0.6477272727272727,
       "cc": 47.79185607418858,
       "cc_by_type": {
        "choice": 57.268927078495125,
        "noul": 26.027397260273965,
        "score": 60.07924388379662
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 47.79185607418858
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3225806451612903,
       "cc": -19.288646333408973,
       "cc_by_type": {
        "choice": 26.153846153846153,
        "noul": -75.0,
        "score": -9.019785154073068
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5873015873015873,
       "cc": 28.891423678941237,
       "cc_by_type": {
        "choice": 56.112852664576806,
        "noul": 8.045977011494255,
        "score": 22.515441360752643
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 28.891423678941237
      },
      "negation_pair_gen": {
       "accuracy": 0.5357142857142857,
       "cc": -27.351155376155678,
       "cc_by_type": {
        "choice": 76.19047619047619,
        "noul": -50.0,
        "score": -108.24394231894323
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.9122807017543859,
       "cc": 88.90122086570477,
       "cc_by_type": {
        "choice": 88.90122086570477
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 88.90122086570477
      },
      "perturb_pair": {
       "accuracy": 0.8333333333333334,
       "cc": 73.07758294752922,
       "cc_by_type": {
        "choice": 76.4089121887287,
        "noul": 71.42857142857142,
        "score": 71.39526522528755
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 73.07758294752922
      },
      "usecase": {
       "accuracy": 0.4978902953586498,
       "cc": 20.903453186109534,
       "cc_by_type": {
        "choice": 45.65426864521104,
        "noul": 1.7543859649122862,
        "score": 15.301704948205275
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 20.903453186109534
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.2727272727272727,
       "cc": -41.39521348113611,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": 0.0,
        "score": -117.51897377674166
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.375,
       "cc": -4.802259887005647,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.5833333333333334,
       "cc": 23.301107918011525,
       "cc_by_type": {
        "choice": 28.994082840236683,
        "noul": -19.999999999999996,
        "score": 60.90924091379788
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 23.301107918011525
      },
      "de": {
       "accuracy": 0.6444444444444445,
       "cc": 44.56362261696139,
       "cc_by_type": {
        "choice": 72.84248642124321,
        "noul": 55.55555555555556,
        "score": 5.292825874085382
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 44.56362261696139
      },
      "el": {
       "accuracy": 0.5,
       "cc": -22.37065677373447,
       "cc_by_type": {
        "choice": 63.855421686746986,
        "noul": -60.0,
        "score": -70.96739200795041
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.5978552278820375,
       "cc": 30.293789733525,
       "cc_by_type": {
        "choice": 56.70218732473357,
        "noul": 3.007518796992481,
        "score": 31.171663078848944
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 30.293789733525
      },
      "es": {
       "accuracy": 0.7142857142857143,
       "cc": 43.83935759486882,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 45.45454545454546,
        "score": -13.936472669939004
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 43.83935759486882
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.4666666666666667,
       "cc": -0.26931923616449654,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": -100.0,
        "score": 49.96778277105517
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.4,
       "cc": 3.9605189196118,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": -19.999999999999996,
        "score": 6.415718249518632
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 3.9605189196118
      },
      "id": {
       "accuracy": 0.5,
       "cc": -16.12903225806452,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 42.698737038013526,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 0.0,
        "score": 28.09621111404058
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 42.698737038013526
      },
      "ja": {
       "accuracy": 0.5,
       "cc": 9.04085560182647,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -42.85714285714286,
        "score": 35.02036006912633
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 9.04085560182647
      },
      "ko": {
       "accuracy": 0.6363636363636364,
       "cc": 51.84513737928095,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": -11.677702616255514
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 51.84513737928095
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 95.1901995158086,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 85.57059854742577
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 95.1901995158086
      },
      "nl": {
       "accuracy": 0.46153846153846156,
       "cc": -6.51647728738407,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0.0,
        "score": -69.4101560961355
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 17.486338797814206,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 17.486338797814206
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 41.28299201238961,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 19.999999999999996,
        "score": 27.460694445564116
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 41.28299201238961
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 76.5780001477811,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 100.0,
        "score": 47.82445270464984
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 76.5780001477811
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 6.320693289140117,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50.0,
        "score": -42.14903124369076
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 6.320693289140117
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 22.124025198251957,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -40.64546826489326
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 22.124025198251957
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": -5.12371977540209,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100.0,
        "score": 49.84623197814155
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.875,
       "cc": 62.994622040580985,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 33.33333333333333,
        "score": 55.65053278840959
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 62.994622040580985
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6653333333333333,
       "cc": 56.64190706512352,
       "cc_by_type": {
        "choice": 56.64190706512352
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 56.64190706512352
      },
      "noul": {
       "accuracy": 0.5226666666666666,
       "cc": 4.533333333333323,
       "cc_by_type": {
        "noul": 4.533333333333323
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 4.533333333333323
      },
      "score": {
       "accuracy": 0.5173333333333333,
       "cc": 27.98616472792205,
       "cc_by_type": {
        "score": 27.98616472792205
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.98616472792205
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5833333333333334,
       "cc": 2.1721868800764432,
       "cc_by_type": {
        "choice": 50.57660626029654,
        "noul": -60.0,
        "score": 15.939954379932786
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 2.1721868800764432
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": 4.537943129820239,
       "cc_by_type": {
        "choice": 1.8003273322422266,
        "noul": 60.00000000000001,
        "score": -48.186497942781514
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 4.537943129820239
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 18.203665219669762,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -25.0,
        "score": 44.12712469126735
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 18.203665219669762
      },
      "demand_forecasting": {
       "accuracy": 0.3076923076923077,
       "cc": -15.30308720258734,
       "cc_by_type": {
        "choice": 27.710843373493972,
        "noul": -50.0,
        "score": -23.620104981255995
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5555555555555556,
       "cc": 31.8743625129997,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 19.999999999999996,
        "score": 31.523708657011518
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 31.8743625129997
      },
      "feature_extraction": {
       "accuracy": 0.5172413793103449,
       "cc": 16.864472729203204,
       "cc_by_type": {
        "choice": 31.623931623931618,
        "noul": -19.999999999999996,
        "score": 38.96948656367799
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 16.864472729203204
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 15.494473495061596,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0.0,
        "score": 11.700811789532606
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 15.494473495061596
      },
      "gaming": {
       "accuracy": 0.3793103448275862,
       "cc": -6.714417440674549,
       "cc_by_type": {
        "choice": 40.59405940594059,
        "noul": -42.85714285714286,
        "score": -17.88016887082138
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.5,
       "cc": 28.798707116861394,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -60.0,
        "score": 94.0151689696318
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 28.798707116861394
      },
      "knowledge_graphs": {
       "accuracy": 0.391304347826087,
       "cc": -0.31602635093334186,
       "cc_by_type": {
        "choice": 35.87786259541985,
        "noul": -60.0,
        "score": 23.174058351780126
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": 1.8570013967353607,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -50.0,
        "score": -28.168833208167896
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 1.8570013967353607
      },
      "legal_compliance": {
       "accuracy": 0.3076923076923077,
       "cc": -26.63894890640322,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": -14.343076227406382
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.7241379310344828,
       "cc": 51.33693366193592,
       "cc_by_type": {
        "choice": 72.4770642201835,
        "noul": 50.0,
        "score": 31.533736765624287
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 51.33693366193592
      },
      "model_routing": {
       "accuracy": 0.68,
       "cc": 48.65467531366116,
       "cc_by_type": {
        "choice": 75.87778075582953,
        "noul": 36.36363636363635,
        "score": 33.72260882151758
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 48.65467531366116
      },
      "moderation_trust_safety": {
       "accuracy": 0.5625,
       "cc": 25.43524834873031,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 19.999999999999996,
        "score": 19.463639783033038
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 25.43524834873031
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 52.73441114090357,
       "cc_by_type": {
        "choice": 84.92462311557789,
        "noul": 33.33333333333333,
        "score": 39.945276973799494
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 52.73441114090357
      },
      "risk_assessment": {
       "accuracy": 0.23076923076923078,
       "cc": -34.30633768347406,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -100.0,
        "score": -8.579390408912758
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.40625,
       "cc": -3.5610027209667976,
       "cc_by_type": {
        "choice": 36.254980079681275,
        "noul": -71.42857142857143,
        "score": 24.490583185989767
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.7272727272727273,
       "cc": 72.3432480209567,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 100.0,
        "score": 31.349553132082498
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 72.3432480209567
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.15231941930905,
      86.38112582622072
     ],
     "I": [
      29.324371200107983,
      38.19996868954735
     ],
     "capability": [
      55.13570575242618,
      61.67190307298117
     ],
     "composite": {
      "A": [
       12.630725624836908,
       23.78332080006789
      ],
      "B": [
       11.12675433052624,
       21.913417722125814
      ],
      "C": [
       8.771337239470075,
       16.516195000047144
      ]
     },
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.49831885164681,
     "capability": 59.981611956556755,
     "intelligence": 34.464905061466695
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6653333333333333,
      "calibration": 85.8172450800459,
      "cc": 56.64190706512352,
      "cc_by_type": {
       "choice": 56.64190706512352
      },
      "cc_tiered": 59.32833079627274,
      "cc_tiered_open": 64.76188322786868,
      "cc_tiered_sealed": 53.894778364676796,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.64190706512352
     },
     "noul": {
      "accuracy": 0.5226666666666666,
      "calibration": 86.51295898861834,
      "cc": 4.533333333333323,
      "cc_by_type": {
       "noul": 4.533333333333323
      },
      "cc_tiered": 13.294340924775709,
      "cc_tiered_open": 26.588681849551417,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 4.533333333333323
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 84.1647524862762,
      "cc": 27.98616472792205,
      "cc_by_type": {
       "score": 27.98616472792205
      },
      "cc_tiered": 30.772043463351643,
      "cc_tiered_open": 36.628943204920006,
      "cc_tiered_sealed": 24.915143721783284,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.98616472792205
     }
    },
    "raw_on_A": {
     "calibration": 83.94383086837013,
     "capability": 57.82491809564761,
     "intelligence": 31.706005322925098
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.624815988733957,
    "Emphasis on Accuracy 60:20:20": 14.535792178016427,
    "Emphasis on Cost 20:20:60": 16.083401791604228,
    "Emphasis on Speed 20:60:20": 20.209563536903357,
    "Intelligence only": 12.230517698051182
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 62.93608607777928,
    "cost": 61.47445744011434,
    "intelligence": 3.0532964296406795,
    "speed": 89.16338093060068
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20193798693071482,
      "ece_score": 59.61240261385703,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 59.61240261385703,
      "score_before_time_limit": 59.61240261385703,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.6799999999999999,
      "brier": 0.28375891613054305,
      "ece": 0.17069524022939242,
      "n": 375,
      "score": 65.86095195412152,
      "score_before_time_limit": 65.86095195412152,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.728,
      "interval90_width": 0.4346666666666666,
      "n": 375,
      "nrps": 0.195768816097399,
      "score": 63.33490366535929,
      "score_before_time_limit": 63.33490366535929,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.2687665552977076
     }
    },
    "score": 62.93608607777928
   },
   "capability": 32.99469125370998,
   "composite_ci95": {
    "A": [
     0.003593558216198542,
     0.23530540138172024
    ],
    "B": [
     0.0023103014112387894,
     0.16353490683027774
    ],
    "C": [
     0.0024955265390267647,
     0.1634065287373057
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: Vega 0.8B is a decision engine on a frozen Qwen3.5-0.8B backbone (Qwen/Qwen3.5-0.8B, resolves in pricing_v15 to deepinfra:Qwen/Qwen3.5-0.8B). DeepInfra list price USD 0.01/M input, USD 0.05/M output from the dated DeepInfra catalog snapshot 2026-09-25 (jevbench-v15-measure-20260925/receipts/DEEPINFRA-MODELS-20260925.json, sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f; source https://api.deepinfra.com/models/list), read by the frozen pricing_v15 on 2026-10-10. The model generates no text: 0 output tokens, cost = input tokens x USD 0.01/M.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.019239100000000002
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega 0.8B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (Kansas City, US), offline after setup (egress cut and proven), reviewed vegaml==0.6.1 wheel, backbone bf16 / engine fp32, context 73,728 (longer states cut on the right), reference PyTorch path for linear attention (no flash-linear-attention / causal-conv1d)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.066515604051393,
    "I_sealed": 2.040077255229966,
    "base": 3.0532964296406795,
    "excess": -8.046806232518286,
    "gap": 2.0264383488214266,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 12.19954681215418,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.378378378378391,
       "hard": 9.405473419314262,
       "judge": 16.94133157547792,
       "standard": 9.585600669736293
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -73.91304347826086,
       "judge": -50.0,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.625135136625836,
       "hard": -0.915170580118807,
       "judge": -9.170003719084164,
       "standard": -19.439472746148546
      }
     },
     "sealed|choice": {
      "cc": 6.1202317656898995,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": -8.874752320367087,
       "hard": 18.874354182697587,
       "judge": 3.2012581807112888,
       "standard": -7.512060647829066
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -23.076923076923073,
       "hard": -69.6969696969697,
       "judge": -58.33333333333333,
       "standard": -78.94736842105263
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": -0.09009355832638732,
       "hard": -3.84788500990465,
       "judge": -4.989943518704232,
       "standard": -15.600158434735768
      }
     }
    }
   },
   "jevbench_score": 0.04021769122022018,
   "key": "vegaml-0.8b",
   "last_measured_on": "2026-10-10",
   "licence": "vegaml package Apache-2.0; HF checkpoint repo declares no licence field (implied, not declared); backbone Qwen3.5 Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-08b-public-intents@73a76a4f6c4fe672e0daa2d45ddc355afc5bc9dd",
   "noul_decisive": {
    "acc_among_decisive": 0.6333333333333333,
    "decisive": 120,
    "decisive_rate": 0.32,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 62.93608607777928,
     "cost": 61.47445744011434,
     "intelligence": 3.0532964296406795,
     "speed": 89.45598874418484
    },
    "board_release": "v1.6.43",
    "capability": 32.99469125370998,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 124,
    "capability_rank": 153,
    "composite_ci95": {
     "A": [
      0.0035937243990094452,
      0.23534462793689395
     ],
     "B": [
      0.00231035635955886,
      0.16355006319549895
     ],
     "C": [
      0.002495641943756559,
      0.16343376940062082
     ]
    },
    "jevbench_score": 0.04022166962784317,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 183,
    "rank": 227,
    "ranks": {
     "A": 227,
     "B": 227,
     "C": 227,
     "capability": 220
    },
    "scores": {
     "A": 0.04022166962784317,
     "B": 0.0266984340300977,
     "C": 0.027931715019335536
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3191188830882311,
     "p50_s_raw": 0.08455944154411554,
     "p95_s_adjusted": 0.35518009765073655,
     "p95_s_raw": 0.10259004882536829
    },
    "views": {
     "Balanced 33:33:33": 0.03151658754618596,
     "Emphasis on Accuracy 60:20:20": 0.018460810494659353,
     "Emphasis on Cost 20:20:60": 0.048117449859240324,
     "Emphasis on Speed 20:60:20": 0.04941520456803552,
     "Intelligence only": 0.011385887829608154
    }
   },
   "rank": 250,
   "ranked": true,
   "ranks": {
    "A": 250,
    "B": 250,
    "C": 250,
    "capability": 242
   },
   "repo": "https://huggingface.co/nandakishorm/vega-08b-public-intents",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.04021769122022018,
    "B": 0.026697031633657865,
    "C": 0.02792895223626402
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3234992638230324,
    "p50_s_raw": 0.08674963191151619,
    "p95_s_adjusted": 0.3747905312106013,
    "p95_s_raw": 0.11239526560530065
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.27586206896551724,
       "cc": -19.36556571427138,
       "cc_by_type": {
        "choice": 25.0,
        "noul": -66.66666666666667,
        "score": -16.43003047614746
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.19642857142857142,
       "cc": -24.988246708550232,
       "cc_by_type": {
        "choice": -4.813664596273289,
        "noul": -76.92307692307692,
        "score": 6.772001393699512
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.15306122448979592,
       "cc": -36.524602578293646,
       "cc_by_type": {
        "choice": -3.7037037037037055,
        "noul": -100.0,
        "score": -5.870104031177226
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.2222222222222222,
       "cc": -26.181811039531453,
       "cc_by_type": {
        "choice": -5.882352941176475,
        "noul": -100.0,
        "score": 27.336919822582118
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0.0
      },
      "jaggedness": {
       "accuracy": 0.2864864864864865,
       "cc": -17.791678833140427,
       "cc_by_type": {
        "choice": 4.12227883279296,
        "noul": -23.076923076923073,
        "score": -34.42039225529116
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.22348484848484848,
       "cc": -10.709273113952321,
       "cc_by_type": {
        "choice": -3.111936832326967,
        "noul": -47.945205479452056,
        "score": 18.929322969922058
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 0.0
      },
      "long_state": {
       "accuracy": 0.1724137931034483,
       "cc": -34.941907856494,
       "cc_by_type": {
        "choice": 4.761904761904764,
        "noul": -81.81818181818181,
        "score": -27.769446513204944
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.41935483870967744,
       "cc": -14.749178737264756,
       "cc_by_type": {
        "choice": 7.692307692307688,
        "noul": 25.0,
        "score": -76.93984390410196
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.2634920634920635,
       "cc": -18.949056458195674,
       "cc_by_type": {
        "choice": 12.225705329153609,
        "noul": -67.81609195402298,
        "score": -1.2567827497176554
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.19642857142857142,
       "cc": -42.402473689854006,
       "cc_by_type": {
        "choice": -9.523809523809524,
        "noul": -60.0,
        "score": -57.6836115457525
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.3157894736842105,
       "cc": 13.42952275249723,
       "cc_by_type": {
        "choice": 13.42952275249723
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 13.42952275249723
      },
      "perturb_pair": {
       "accuracy": 0.45,
       "cc": 6.179349639681301,
       "cc_by_type": {
        "choice": 29.22673656618611,
        "noul": -14.28571428571429,
        "score": 3.5970266385720873
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 6.179349639681301
      },
      "usecase": {
       "accuracy": 0.29535864978902954,
       "cc": -19.00704443525642,
       "cc_by_type": {
        "choice": 21.37213335902873,
        "noul": -71.9298245614035,
        "score": -6.46344210339449
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -7.714580910237068,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": 0.0,
        "score": -16.47707606404454
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.25,
       "cc": -38.135593220338976,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.2916666666666667,
       "cc": -23.813705445205983,
       "cc_by_type": {
        "choice": 5.325443786982243,
        "noul": -100.0,
        "score": 23.23343987739981
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.26666666666666666,
       "cc": -9.48435284148812,
       "cc_by_type": {
        "choice": 13.095956547978272,
        "noul": -33.333333333333336,
        "score": -8.215681739109293
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0.0
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": -38.08299402731068,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -60.0,
        "score": -94.00801822651034
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.2529043789097408,
       "cc": -19.4845221445307,
       "cc_by_type": {
        "choice": 6.449803701626394,
        "noul": -60.902255639097746,
        "score": -4.00111449612075
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.35714285714285715,
       "cc": -3.020542459003753,
       "cc_by_type": {
        "choice": -17.81818181818182,
        "noul": 9.090909090909083,
        "score": -0.3343546497385219
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 1.6431924882629083,
       "cc_by_type": {
        "choice": 36.61971830985915,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 1.6431924882629083
      },
      "fr": {
       "accuracy": 0.06666666666666667,
       "cc": -42.623688574546726,
       "cc_by_type": {
        "choice": -26.93935119887165,
        "noul": -100.0,
        "score": -0.9317145247685232
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.2,
       "cc": -28.83602734828803,
       "cc_by_type": {
        "choice": -11.801242236024846,
        "noul": -40.0,
        "score": -34.70683980883924
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.5294117647058824,
       "cc": 5.113250586139728,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 0.0,
        "score": -33.88450776203216
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 5.113250586139728
      },
      "ja": {
       "accuracy": 0.21875,
       "cc": -21.637722673747195,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -71.42857142857143,
        "score": 4.076379017085941
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.18181818181818182,
       "cc": -20.186550397257573,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 0.0,
        "score": -62.1989954540678
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.2727272727272727,
       "cc": -1.2255967997920887,
       "cc_by_type": {
        "choice": -9.890109890109892,
        "noul": -50.0,
        "score": 56.213319490733625
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "nl": {
       "accuracy": 0.07692307692307693,
       "cc": -33.67879710336313,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": -100.0,
        "score": -0.7578397780559687
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -65.57377049180329,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.275,
       "cc": -16.97531167960543,
       "cc_by_type": {
        "choice": 29.164844774814167,
        "noul": -80.0,
        "score": -0.0907798136304594
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 21.729761287554258,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": -66.66666666666667,
        "score": 49.946402790636014
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 21.729761287554258
      },
      "sv": {
       "accuracy": 0.25,
       "cc": -8.670532037806744,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": 62.87729277546865
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.18181818181818182,
       "cc": -58.40033972849334,
       "cc_by_type": {
        "choice": -5.26315789473684,
        "noul": -100.0,
        "score": -69.93786129074316
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.18181818181818182,
       "cc": -43.03199883446496,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": -42.13947476426443
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 18.615495552140683,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": -33.333333333333336,
        "score": 54.04468485462024
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.615495552140683
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.2866666666666667,
       "cc": 7.583347728450512,
       "cc_by_type": {
        "choice": 7.583347728450512
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 7.583347728450512
      },
      "noul": {
       "accuracy": 0.20266666666666666,
       "cc": -59.46666666666667,
       "cc_by_type": {
        "noul": -59.46666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.264,
       "cc": -4.914138326410811,
       "cc_by_type": {
        "score": -4.914138326410811
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.25,
       "cc": -42.23115369530359,
       "cc_by_type": {
        "choice": 11.037891268533771,
        "noul": -100.0,
        "score": -37.73135235444454
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 10.053783668961861,
       "cc_by_type": {
        "choice": 21.440261865793783,
        "noul": -19.999999999999996,
        "score": 28.721089141091795
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 10.053783668961861
      },
      "customer_support": {
       "accuracy": 0.25,
       "cc": -14.695878083746338,
       "cc_by_type": {
        "choice": -12.90322580645161,
        "noul": -50.0,
        "score": 18.815591555212595
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.07692307692307693,
       "cc": -52.52931897614329,
       "cc_by_type": {
        "choice": -20.481927710843376,
        "noul": -100.0,
        "score": -37.10602921758648
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.1111111111111111,
       "cc": -38.90478330755122,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -100.0,
        "score": -23.546648059299624
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.3793103448275862,
       "cc": -13.328851245398967,
       "cc_by_type": {
        "choice": 31.623931623931618,
        "noul": -60.0,
        "score": -11.61048536012852
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.34615384615384615,
       "cc": -18.503379574597428,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": -75.0,
        "score": -28.33622568031402
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -25.77506638432793,
       "cc_by_type": {
        "choice": -6.9306930693069315,
        "noul": -71.42857142857143,
        "score": 1.0340653448945747
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.14285714285714285,
       "cc": -30.119048798099303,
       "cc_by_type": {
        "choice": -11.11111111111111,
        "noul": -100.0,
        "score": 20.7539647168132
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.17391304347826086,
       "cc": -34.69531570651282,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100.0,
        "score": -3.3225883409125245
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 8.261797778580346,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -25.0,
        "score": -17.694281461006934
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 8.261797778580346
      },
      "legal_compliance": {
       "accuracy": 0.23076923076923078,
       "cc": -31.134132674360444,
       "cc_by_type": {
        "choice": 18.032786885245905,
        "noul": -100.0,
        "score": -11.435184908327244
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.3103448275862069,
       "cc": -9.67272641203819,
       "cc_by_type": {
        "choice": 17.431192660550458,
        "noul": -75.0,
        "score": 28.550628103334972
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.29333333333333333,
       "cc": 2.0858721093715658,
       "cc_by_type": {
        "choice": 1.9029750737067974,
        "noul": -18.181818181818176,
        "score": 22.536459436226075
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 2.0858721093715658
      },
      "moderation_trust_safety": {
       "accuracy": 0.3125,
       "cc": -22.66548316017918,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100.0,
        "score": -4.8385547436954335
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.38095238095238093,
       "cc": 1.0762044705122094,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": -66.66666666666667,
        "score": 0.046033847047532994
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 1.0762044705122094
      },
      "risk_assessment": {
       "accuracy": 0.3076923076923077,
       "cc": -21.111851703106247,
       "cc_by_type": {
        "choice": 24.528301886792452,
        "noul": -100.0,
        "score": 12.136143003888812
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.1875,
       "cc": -22.40823864613203,
       "cc_by_type": {
        "choice": 4.382470119521911,
        "noul": -71.42857142857143,
        "score": -0.17861462934656291
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.2727272727272727,
       "cc": -35.0856709518421,
       "cc_by_type": {
        "choice": 14.08114558472553,
        "noul": -66.66666666666667,
        "score": -52.671491773585146
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      57.40975035799149,
      66.49290908060016
     ],
     "I": [
      1.3348314800373893,
      5.689355834192679
     ],
     "capability": [
      29.933677735120224,
      35.37515714416046
     ],
     "composite": {
      "A": [
       0.003593558216198542,
       0.23530540138172024
      ],
      "B": [
       0.0023103014112387894,
       0.16353490683027774
      ],
      "C": [
       0.0024955265390267647,
       0.1634065287373057
      ]
     },
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.93608607777928,
     "capability": 32.99469125370998,
     "intelligence": 3.0532964296406795
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2866666666666667,
      "calibration": 59.61240261385703,
      "cc": 7.583347728450512,
      "cc_by_type": {
       "choice": 7.583347728450512
      },
      "cc_tiered": 9.15988928892204,
      "cc_tiered_open": 12.19954681215418,
      "cc_tiered_sealed": 6.1202317656898995,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 7.583347728450512
     },
     "noul": {
      "accuracy": 0.20266666666666666,
      "calibration": 65.86095195412152,
      "cc": -59.46666666666667,
      "cc_by_type": {
       "noul": -59.46666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.264,
      "calibration": 63.33490366535929,
      "cc": -4.914138326410811,
      "cc_by_type": {
       "score": -4.914138326410811
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 59.68341710343884,
     "capability": 30.94517056072779,
     "intelligence": 2.206924018016739
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.03151333064727078,
    "Emphasis on Accuracy 60:20:20": 0.0184601399807529,
    "Emphasis on Cost 20:20:60": 0.048112894882889716,
    "Emphasis on Speed 20:60:20": 0.04940079543571209,
    "Intelligence only": 0.011385887829608154
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 57.46108428162757,
    "cost": 47.16081979852447,
    "intelligence": 15.69626515886651,
    "speed": 83.93992701495202
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2574125103434302,
      "ece_score": 48.517497931313954,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 47.80590796165468,
      "score_before_time_limit": 48.517497931313954,
      "time_limit_failed_share": 0.014666666666666666
     },
     "noul": {
      "abstention_rate": 0.30400000000000005,
      "brier": 0.2781281471262366,
      "ece": 0.2285300562531464,
      "n": 375,
      "score": 52.701365079389184,
      "score_before_time_limit": 54.293988749370726,
      "time_limit_failed_share": 0.029333333333333333
     },
     "score": {
      "interval90_coverage": 0.8346666666666667,
      "interval90_width": 0.49288888888888904,
      "n": 375,
      "nrps": 0.16226214991492005,
      "score": 71.87597980383885,
      "score_before_time_limit": 73.24318594141187,
      "time_limit_failed_share": 0.018666666666666668,
      "top_ece": 0.18643706562842138
     }
    },
    "score": 57.46108428162757
   },
   "capability": 36.57867472024704,
   "composite_ci95": {
    "A": [
     1.283761083400865,
     5.7525348295903695
    ],
    "B": [
     0.9796330050681419,
     4.792240387846222
    ],
    "C": [
     0.891500752361712,
     3.994815853882201
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: Vega 4B is a decision engine on a frozen Qwen3.5-4B backbone (Qwen/Qwen3.5-4B, resolves in pricing_v15 to deepinfra:Qwen/Qwen3.5-4B). DeepInfra list price USD 0.03/M input, USD 0.15/M output from the dated DeepInfra catalog snapshot 2026-09-25 (jevbench-v15-measure-20260925/receipts/DEEPINFRA-MODELS-20260925.json, sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f; source https://api.deepinfra.com/models/list), read by the frozen pricing_v15 on 2026-10-10. The listing is flagged by pricing_v15 as deprecated after the 25 Sep cut-off (DEPRECATED_AFTER_CUTOFF) and stays frozen with that note. The model generates no text: 0 output tokens, cost = input tokens x USD 0.03/M.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.057717300000000006
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 29,
    "n_over_limit_answered_ok": 29,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega 4B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (Des Moines, US), offline after setup (egress cut and proven), reviewed vegaml==0.6.1 wheel, backbone bf16 / engine fp32, context 73,728 (longer states cut on the right), reference PyTorch path for linear attention (no flash-linear-attention / causal-conv1d)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 21.771877584149628,
    "I_sealed": 9.620652733583393,
    "base": 15.69626515886651,
    "excess": 2.077980269226522,
    "gap": 12.151224850566235,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 41.396463388934684,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 45.51351351351353,
       "hard": 28.279333123623786,
       "judge": 48.58272907053395,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 9.665976535541756,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 14.253192827972441,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.64297240164319,
       "hard": 6.501670939484594,
       "judge": 20.791162375101624,
       "standard": 8.754392497418984
      }
     },
     "sealed|choice": {
      "cc": 24.831311180753584,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 32.4225675252894,
       "hard": 22.337034187094638,
       "judge": 22.378367409060942,
       "standard": 29.70365265334253
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 7.692307692307687,
       "hard": -28.787878787878785,
       "judge": 8.333333333333325,
       "standard": -12.28070175438597
      }
     },
     "sealed|score": {
      "cc": 4.030647019996595,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 23.787017462891814,
       "hard": -4.6189235136853135,
       "judge": 10.43509835324301,
       "standard": 1.8449258660431789
      }
     }
    }
   },
   "jevbench_score": 3.0701231066291896,
   "key": "vegaml-4b",
   "last_measured_on": "2026-10-10",
   "licence": "vegaml package Apache-2.0; HF checkpoint repo declares no licence field (implied, not declared); backbone Qwen3.5 Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-08b-public-intents@73a76a4f6c4fe672e0daa2d45ddc355afc5bc9dd",
   "noul_decisive": {
    "acc_among_decisive": 0.696,
    "decisive": 250,
    "decisive_rate": 0.6868131868131868,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 58.68489087403218,
     "cost": 47.16081979852447,
     "intelligence": 16.689214224692094,
     "speed": 84.39386641436894
    },
    "board_release": "v1.6.43",
    "capability": 37.687052549362136,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 118,
    "capability_rank": 146,
    "composite_ci95": {
     "A": [
      1.5678216817255275,
      6.594029572003121
     ],
     "B": [
      1.2095817426628035,
      5.54602239255017
     ],
     "C": [
      1.0887650567538385,
      4.579187202779947
     ]
    },
    "jevbench_score": 3.603902446043355,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 148,
    "rank": 188,
    "ranks": {
     "A": 188,
     "B": 191,
     "C": 189,
     "capability": 210
    },
    "scores": {
     "A": 3.603902446043355,
     "B": 2.9164270830443,
     "C": 2.502710031974551
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5304726889356971,
     "p50_s_raw": 0.19023634446784854,
     "p95_s_adjusted": 0.6854098306968808,
     "p95_s_raw": 0.2677049153484404
    },
    "views": {
     "Balanced 33:33:33": 3.198325647386112,
     "Emphasis on Accuracy 60:20:20": 2.3288005981136943,
     "Emphasis on Cost 20:20:60": 3.6607417516659844,
     "Emphasis on Speed 20:60:20": 4.247790339721065,
     "Intelligence only": 1.654208859284027
    }
   },
   "rank": 216,
   "ranked": true,
   "ranks": {
    "A": 216,
    "B": 217,
    "C": 217,
    "capability": 235
   },
   "repo": "https://huggingface.co/nandakishorm/vega-08b-public-intents",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.0701231066291896,
    "B": 2.463619140446955,
    "C": 2.13202993515916
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5442217165604234,
    "p50_s_raw": 0.1971108582802117,
    "p95_s_adjusted": 0.7417053825221955,
    "p95_s_raw": 0.29585269126109776
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5862068965517241,
       "cc": 32.10254092129096,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 0.0,
        "score": 46.30762276387288
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 32.10254092129096
      },
      "calibration_gen": {
       "accuracy": 0.3482142857142857,
       "cc": -7.764599070714067,
       "cc_by_type": {
        "choice": 16.14906832298137,
        "noul": -38.46153846153846,
        "score": -0.9813270735851143
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.35714285714285715,
       "cc": 4.1474482072018475,
       "cc_by_type": {
        "choice": 7.407407407407411,
        "noul": -7.14285714285714,
        "score": 12.177794357055271
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 4.1474482072018475
      },
      "fairness": {
       "accuracy": 0.7037037037037037,
       "cc": 46.819844941220424,
       "cc_by_type": {
        "choice": 64.70588235294117,
        "noul": 0.0,
        "score": 75.75365247072008
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 46.819844941220424
      },
      "jaggedness": {
       "accuracy": 0.34054054054054056,
       "cc": -9.581651647472881,
       "cc_by_type": {
        "choice": 19.407132931912926,
        "noul": -28.205128205128204,
        "score": -19.946959669203366
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.5303030303030303,
       "cc": 35.11456970853259,
       "cc_by_type": {
        "choice": 34.9744542498839,
        "noul": 28.767123287671236,
        "score": 41.602131588042646
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 35.11456970853259
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3548387096774194,
       "cc": -22.26927167429365,
       "cc_by_type": {
        "choice": 35.38461538461538,
        "noul": -75.0,
        "score": -27.192430407496325
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.4603174603174603,
       "cc": 15.9575022436564,
       "cc_by_type": {
        "choice": 35.512763098970005,
        "noul": -1.1494252873563204,
        "score": 13.509168919355519
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 15.9575022436564
      },
      "negation_pair_gen": {
       "accuracy": 0.32142857142857145,
       "cc": -18.937881655648187,
       "cc_by_type": {
        "choice": -9.523809523809524,
        "noul": 0.0,
        "score": -47.28983544313503
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.5614035087719298,
       "cc": 44.50610432852386,
       "cc_by_type": {
        "choice": 44.50610432852386
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 44.50610432852386
      },
      "perturb_pair": {
       "accuracy": 0.5333333333333333,
       "cc": 41.1419841739421,
       "cc_by_type": {
        "choice": 25.29488859764089,
        "noul": 42.85714285714286,
        "score": 55.273921067042544
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 41.1419841739421
      },
      "usecase": {
       "accuracy": 0.379746835443038,
       "cc": -1.691456792175881,
       "cc_by_type": {
        "choice": 28.30988629793796,
        "noul": -22.807017543859654,
        "score": -10.577239130605953
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5454545454545454,
       "cc": -14.905334686220934,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -50.0,
        "score": -68.04933739199615
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.375,
       "cc": -4.802259887005647,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.4583333333333333,
       "cc": 22.143800617963134,
       "cc_by_type": {
        "choice": 64.49704142011835,
        "noul": 19.999999999999996,
        "score": -18.065639566228953
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 22.143800617963134
      },
      "de": {
       "accuracy": 0.4444444444444444,
       "cc": 3.742298256027752,
       "cc_by_type": {
        "choice": 29.390464695232346,
        "noul": -33.333333333333336,
        "score": 15.169763406184245
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 3.742298256027752
      },
      "el": {
       "accuracy": 0.5,
       "cc": -7.267096986083452,
       "cc_by_type": {
        "choice": 51.80722891566265,
        "noul": -60.0,
        "score": -13.608519873913005
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.42001787310098304,
       "cc": 7.619934970948897,
       "cc_by_type": {
        "choice": 25.96747055524391,
        "noul": -8.270676691729328,
        "score": 5.163011049332111
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 7.619934970948897
      },
      "es": {
       "accuracy": 0.5357142857142857,
       "cc": 23.735811235182705,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": 9.090909090909083,
        "score": 14.480160978275391
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 23.735811235182705
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -19.483568075117372,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5333333333333333,
       "cc": 9.81849210967231,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": -50.0,
        "score": 30.231216808565588
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 9.81849210967231
      },
      "hi": {
       "accuracy": 0.36666666666666664,
       "cc": -2.9723800154374516,
       "cc_by_type": {
        "choice": 37.888198757763966,
        "noul": -40.0,
        "score": -6.805338804076322
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -14.516129032258062,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.5882352941176471,
       "cc": 41.890588956237984,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50.0,
        "score": 1.059637108488276
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 41.890588956237984
      },
      "ja": {
       "accuracy": 0.34375,
       "cc": 6.462836027337623,
       "cc_by_type": {
        "choice": -5.691056910569103,
        "noul": -14.28571428571429,
        "score": 39.36527927829626
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 6.462836027337623
      },
      "ko": {
       "accuracy": 0.36363636363636365,
       "cc": 18.2762217539951,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 0.0,
        "score": 20.402435753788563
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 18.2762217539951
      },
      "mixed": {
       "accuracy": 0.7272727272727273,
       "cc": 59.83071388979721,
       "cc_by_type": {
        "choice": 56.04395604395604,
        "noul": 100.0,
        "score": 23.448185625435602
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 59.83071388979721
      },
      "nl": {
       "accuracy": 0.15384615384615385,
       "cc": -18.26355956150215,
       "cc_by_type": {
        "choice": -25.34818941504178,
        "noul": -66.66666666666667,
        "score": 37.224177397202
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.525,
       "cc": 43.18418848568871,
       "cc_by_type": {
        "choice": 68.51770878880629,
        "noul": 60.00000000000001,
        "score": 1.0348566682598204
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 43.18418848568871
      },
      "pt": {
       "accuracy": 0.4444444444444444,
       "cc": 15.083741051175762,
       "cc_by_type": {
        "choice": 27.638190954773865,
        "noul": -33.333333333333336,
        "score": 50.94636553208676
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 15.083741051175762
      },
      "sv": {
       "accuracy": 0.125,
       "cc": -24.998495011569247,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -50.0,
        "score": 8.337848298625605
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 20.74230849140807,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": 100.0,
        "score": -85.14149557840736
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 20.74230849140807
      },
      "uk": {
       "accuracy": 0.18181818181818182,
       "cc": -41.58877432033782,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": -37.80980122188302
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 65.40226528868891,
       "cc_by_type": {
        "choice": 67.56756756756756,
        "noul": 33.33333333333333,
        "score": 95.30589496516582
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.40226528868891
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.444,
       "cc": 27.966833650025915,
       "cc_by_type": {
        "choice": 27.966833650025915
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 27.966833650025915
      },
      "noul": {
       "accuracy": 0.464,
       "cc": -7.199999999999996,
       "cc_by_type": {
        "noul": -7.199999999999996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.35733333333333334,
       "cc": 6.1512696164423115,
       "cc_by_type": {
        "score": 6.1512696164423115
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 6.1512696164423115
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3333333333333333,
       "cc": -35.1534607494542,
       "cc_by_type": {
        "choice": 20.922570016474467,
        "noul": -100.0,
        "score": -26.382952264837066
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 15.916918386842573,
       "cc_by_type": {
        "choice": 41.08019639934533,
        "noul": 19.999999999999996,
        "score": -13.329441238817608
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 15.916918386842573
      },
      "customer_support": {
       "accuracy": 0.45,
       "cc": 8.926626674691754,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 25.0,
        "score": -33.703990943666675
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 8.926626674691754
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -37.51418716919469,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100.0,
        "score": -16.15701933890936
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2222222222222222,
       "cc": -13.643840746791454,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -60.0,
        "score": -6.397360749691128
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.27586206896551724,
       "cc": -6.646648832243309,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -19.999999999999996,
        "score": -5.922852479635909
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6538461538461539,
       "cc": 30.68478198913924,
       "cc_by_type": {
        "choice": 86.95652173913045,
        "noul": 25.0,
        "score": -19.902175771712738
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 30.68478198913924
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -8.539638885229605,
       "cc_by_type": {
        "choice": -6.9306930693069315,
        "noul": -42.85714285714286,
        "score": 24.16891927076098
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.21428571428571427,
       "cc": -26.327496496464544,
       "cc_by_type": {
        "choice": 4.761904761904763,
        "noul": -60.0,
        "score": -23.744394251298395
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.21739130434782608,
       "cc": -7.707530000989269,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -19.999999999999996,
        "score": -2.359231224341851
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 19.542316672389997,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 25.0,
        "score": -17.592562177951955
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 19.542316672389997
      },
      "legal_compliance": {
       "accuracy": 0.23076923076923078,
       "cc": -17.21253914360014,
       "cc_by_type": {
        "choice": 18.032786885245905,
        "noul": -100.0,
        "score": 30.32959568395368
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.41379310344827586,
       "cc": -0.8829038829667036,
       "cc_by_type": {
        "choice": 35.77981651376147,
        "noul": -50.0,
        "score": 11.571471837338422
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.5,
       "cc": 24.299647249468666,
       "cc_by_type": {
        "choice": 42.106673813990895,
        "noul": 22.72727272727273,
        "score": 8.064995207142367
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 24.299647249468666
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 21.15669006850672,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 19.999999999999996,
        "score": 22.417438626572796
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 21.15669006850672
      },
      "recruiting": {
       "accuracy": 0.42857142857142855,
       "cc": 4.28970309222692,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": -33.333333333333336,
        "score": -23.646803621141665
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 4.28970309222692
      },
      "risk_assessment": {
       "accuracy": 0.5384615384615384,
       "cc": 13.524055414252595,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -36.15739350566989
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 13.524055414252595
      },
      "scientific_discovery": {
       "accuracy": 0.21875,
       "cc": -20.974690029781613,
       "cc_by_type": {
        "choice": 20.31872509960159,
        "noul": -71.42857142857143,
        "score": -11.814223760374997
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.5,
       "cc": 2.15502085448014,
       "cc_by_type": {
        "choice": 57.04057279236276,
        "noul": -33.333333333333336,
        "score": -17.242176895589
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 2.15502085448014
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      52.22926067223202,
      61.459644474882964
     ],
     "I": [
      11.222571232188471,
      20.06206658675151
     ],
     "capability": [
      32.61433592571537,
      39.87642689527607
     ],
     "composite": {
      "A": [
       1.283761083400865,
       5.7525348295903695
      ],
      "B": [
       0.9796330050681419,
       4.792240387846222
      ],
      "C": [
       0.891500752361712,
       3.994815853882201
      ]
     },
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 57.46108428162757,
     "capability": 36.57867472024704,
     "intelligence": 15.69626515886651
    },
    "per_type": {
     "choice": {
      "accuracy": 0.444,
      "calibration": 47.80590796165468,
      "cc": 27.966833650025915,
      "cc_by_type": {
       "choice": 27.966833650025915
      },
      "cc_tiered": 33.11388728484413,
      "cc_tiered_open": 41.396463388934684,
      "cc_tiered_sealed": 24.831311180753584,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 27.966833650025915
     },
     "noul": {
      "accuracy": 0.464,
      "calibration": 52.701365079389184,
      "cc": -7.199999999999996,
      "cc_by_type": {
       "noul": -7.199999999999996
      },
      "cc_tiered": 4.832988267770878,
      "cc_tiered_open": 9.665976535541756,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.35733333333333334,
      "calibration": 71.87597980383885,
      "cc": 6.1512696164423115,
      "cc_by_type": {
       "score": 6.1512696164423115
      },
      "cc_tiered": 9.141919923984517,
      "cc_tiered_open": 14.253192827972441,
      "cc_tiered_sealed": 4.030647019996595,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 6.1512696164423115
     }
    },
    "raw_on_A": {
     "calibration": 59.42725004267636,
     "capability": 37.79076765897078,
     "intelligence": 16.15428527526519
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.7164470339746303,
    "Emphasis on Accuracy 60:20:20": 1.9548859019634979,
    "Emphasis on Cost 20:20:60": 3.1484546409137137,
    "Emphasis on Speed 20:60:20": 3.6333404166707344,
    "Intelligence only": 1.376168631913456
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "org2ai",
   "axes": {
    "calibration": 86.58998638479137,
    "cost": 46.287007162302075,
    "intelligence": 51.951877017848766,
    "speed": 92.782872129877
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05465848238633132,
      "ece_score": 89.06830352273374,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 89.06830352273374,
      "score_before_time_limit": 89.06830352273374,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.16249185347848963,
      "ece": 0.09805762809316519,
      "n": 375,
      "score": 80.38847438136696,
      "score_before_time_limit": 80.38847438136696,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9813333333333333,
      "interval90_width": 0.6019999999999996,
      "n": 375,
      "nrps": 0.1056056821458286,
      "score": 90.31318125027343,
      "score_before_time_limit": 90.31318125027343,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04406534642435141
     }
    },
    "score": 86.58998638479137
   },
   "capability": 69.27093170132007,
   "composite_ci95": {
    "A": [
     45.32939136901336,
     55.42668230522751
    ],
    "B": [
     42.74044451252099,
     53.75204709495266
    ],
    "C": [
     31.478744006259276,
     48.301452339916715
    ]
   },
   "cost": {
    "basis": "Self-hosted open weights, no bookable tariff. v1.5-M2 base-model reference rule: base model Qwen/Qwen3.5-4B (model card base_model) -> DeepInfra Qwen/Qwen3.5-4B, USD 0.03/M input and USD 0.15/M output, frozen 25 Sep 2026 price cut-off snapshot (METHOD-v1.5-ADDENDUM-PRICING-INTERPRETATION-1 I-2/I-3; listing deprecated after the cut-off, stays the frozen v1.5 reference). Same basis as the published Wald 4B (v2.0) row. Read 2026-10-10.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06172102
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Wald 4B v2.1",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX PRO 6000 96GB), offline read-only container (--network none); official vllm/vllm-openai:v0.30.0@sha256:8a69ffad... image; author's server wald_serve.native_vision from the model repo, --prompt-format plain (author RUNBOOK's JevBench format), serving.json defaults otherwise (effort none = one pass, noul_decisive_floor 0.81)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 56.69080677700682,
    "I_sealed": 47.21294725869071,
    "base": 51.951877017848766,
    "excess": -0.5953850630236044,
    "gap": 9.477859518316109,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.9521688095628,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243245,
       "hard": 64.1396665618119,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 66.16425120772948,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 32.9560003137282,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 42.310346490056276,
       "hard": 31.56850185540615,
       "judge": 32.18036106421194,
       "standard": 32.21728301648263
      }
     },
     "sealed|choice": {
      "cc": 55.12375685402749,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 62.45698195849411,
       "hard": 49.543805650214345,
       "judge": 57.07980315559839,
       "standard": 59.6829772570641
      }
     },
     "sealed|noul": {
      "cc": 53.117408906882595,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 48.71794871794872,
       "hard": 50.0,
       "judge": 55.55555555555556,
       "standard": 57.89473684210527
      }
     },
     "sealed|score": {
      "cc": 33.39767601516204,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 38.95455244599806,
       "hard": 28.60554068871497,
       "judge": 38.85138751678833,
       "standard": 32.02294120019875
      }
     }
    }
   },
   "jevbench_score": 54.2578218733026,
   "key": "wald-4b-v2-1",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (weights and server code)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "org2ai/Wald-4B@ce3286f7efd332530e5dccd8a17d6496e029f714",
   "noul_decisive": {
    "acc_among_decisive": 0.7813333333333333,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.58998638479137,
     "cost": 46.287007162302075,
     "intelligence": 51.951877017848766,
     "speed": 93.07746270925374
    },
    "board_release": "v1.6.43",
    "capability": 69.27093170132007,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 29,
    "capability_rank": 50,
    "composite_ci95": {
     "A": [
      45.35289667440805,
      55.4572699256173
     ],
     "B": [
      42.75715999124745,
      53.77505802688884
     ],
     "C": [
      31.495067135005595,
      48.32811751847993
     ]
    },
    "jevbench_score": 54.28713266716134,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 38,
    "rank": 62,
    "ranks": {
     "A": 62,
     "B": 62,
     "C": 63,
     "capability": 84
    },
    "scores": {
     "A": 54.28713266716134,
     "B": 52.0059503801659,
     "C": 40.700232429112376
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20302085168659686,
     "p50_s_raw": 0.026510425843298435,
     "p95_s_adjusted": 0.24250074867159127,
     "p95_s_raw": 0.046250374335795645
    },
    "views": {
     "Balanced 33:33:33": 49.82850819150803,
     "Emphasis on Accuracy 60:20:20": 47.56124181789482,
     "Emphasis on Cost 20:20:60": 45.197608867054875,
     "Emphasis on Speed 20:60:20": 58.630700497081925,
     "Intelligence only": 44.522489120413574
    }
   },
   "rank": 67,
   "ranked": true,
   "ranks": {
    "A": 67,
    "B": 68,
    "C": 74,
    "capability": 98
   },
   "repo": "https://huggingface.co/org2ai/Wald-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 54.2578218733026,
    "B": 51.98442828149203,
    "C": 40.67825749575116
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20795744713395833,
    "p50_s_raw": 0.02897872356697917,
    "p95_s_adjusted": 0.253360151220113,
    "p95_s_raw": 0.05168007561005651
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7586206896551724,
       "cc": 63.68096433104207,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 66.66666666666667,
        "score": 49.376226326459516
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 63.68096433104207
      },
      "calibration_gen": {
       "accuracy": 0.6160714285714286,
       "cc": 44.10064431077017,
       "cc_by_type": {
        "choice": 46.42857142857144,
        "noul": 61.53846153846154,
        "score": 24.334899965277536
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 44.10064431077017
      },
      "composite_gen": {
       "accuracy": 0.6836734693877551,
       "cc": 57.621281961194256,
       "cc_by_type": {
        "choice": 74.07407407407408,
        "noul": 64.28571428571428,
        "score": 34.50405752379443
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 57.621281961194256
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 89.90132796385994,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 69.70398389157982
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 89.90132796385994
      },
      "jaggedness": {
       "accuracy": 0.5459459459459459,
       "cc": 28.653270418657403,
       "cc_by_type": {
        "choice": 47.19777674849467,
        "noul": 12.82051282051282,
        "score": 25.94152168696473
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 28.653270418657403
      },
      "llm_router": {
       "accuracy": 0.7424242424242424,
       "cc": 68.06655796712668,
       "cc_by_type": {
        "choice": 56.33999071063632,
        "noul": 83.56164383561644,
        "score": 64.29803935512727
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 68.06655796712668
      },
      "long_state": {
       "accuracy": 0.3793103448275862,
       "cc": 4.748724836384873,
       "cc_by_type": {
        "choice": 40.476190476190474,
        "noul": -27.27272727272727,
        "score": 1.0427113056914128
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 4.748724836384873
      },
      "ml_usecase": {
       "accuracy": 0.5161290322580645,
       "cc": 22.614498009598538,
       "cc_by_type": {
        "choice": 35.38461538461538,
        "noul": 50.0,
        "score": -17.541121355819755
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 22.614498009598538
      },
      "multilingual": {
       "accuracy": 0.6412698412698413,
       "cc": 46.51177232484513,
       "cc_by_type": {
        "choice": 55.217196596506945,
        "noul": 58.62068965517242,
        "score": 25.69743072285603
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 46.51177232484513
      },
      "negation_pair_gen": {
       "accuracy": 0.6964285714285714,
       "cc": 33.02311941019134,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": -12.835403674187873
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 33.02311941019134
      },
      "option_order": {
       "accuracy": 0.8245614035087719,
       "cc": 77.80244173140953,
       "cc_by_type": {
        "choice": 77.80244173140953
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 77.80244173140953
      },
      "perturb_pair": {
       "accuracy": 0.8666666666666667,
       "cc": 84.22071661549569,
       "cc_by_type": {
        "choice": 76.4089121887287,
        "noul": 100.0,
        "score": 76.25323765775835
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 84.22071661549569
      },
      "usecase": {
       "accuracy": 0.6033755274261603,
       "cc": 41.10751808936778,
       "cc_by_type": {
        "choice": 57.217190210059755,
        "noul": 43.85964912280702,
        "score": 22.24571493523657
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 41.10751808936778
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8181818181818182,
       "cc": 42.61103847074362,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 50.0,
        "score": 4.49978207889753
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 42.61103847074362
      },
      "cs": {
       "accuracy": 0.625,
       "cc": 61.864406779661024,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": 100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 61.864406779661024
      },
      "da": {
       "accuracy": 0.5416666666666666,
       "cc": 42.74539505383657,
       "cc_by_type": {
        "choice": 28.994082840236683,
        "noul": 60.00000000000001,
        "score": 39.24210232127302
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 42.74539505383657
      },
      "de": {
       "accuracy": 0.7333333333333333,
       "cc": 55.337376093377294,
       "cc_by_type": {
        "choice": 78.27398913699457,
        "noul": 55.55555555555556,
        "score": 32.18258358758174
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 55.337376093377294
      },
      "el": {
       "accuracy": 0.5555555555555556,
       "cc": 11.934924140935678,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": 60.00000000000001,
        "score": -63.954263721771284
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 11.934924140935678
      },
      "en": {
       "accuracy": 0.6657730116175157,
       "cc": 49.96242193822483,
       "cc_by_type": {
        "choice": 59.169938306225426,
        "noul": 53.38345864661653,
        "score": 37.33386886183252
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 49.96242193822483
      },
      "es": {
       "accuracy": 0.8571428571428571,
       "cc": 71.78869471940175,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 100.0,
        "score": 41.54790234002343
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 71.78869471940175
      },
      "fi": {
       "accuracy": 0.7777777777777778,
       "cc": 78.87323943661971,
       "cc_by_type": {
        "choice": 57.74647887323943,
        "noul": 100.0
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 78.87323943661971
      },
      "fr": {
       "accuracy": 0.7333333333333333,
       "cc": 53.603908086020034,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50.0,
        "score": 36.19959449783442
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 53.603908086020034
      },
      "hi": {
       "accuracy": 0.43333333333333335,
       "cc": 25.416831317917744,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": 60.00000000000001,
        "score": 3.2070156928836524
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 25.416831317917744
      },
      "id": {
       "accuracy": 0.6666666666666666,
       "cc": 33.87096774193548,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 33.87096774193548
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 54.2758437211133,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50.0,
        "score": 38.21540140311422
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 54.2758437211133
      },
      "ja": {
       "accuracy": 0.65625,
       "cc": 49.17418342165371,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 71.42857142857142,
        "score": 24.874466641267755
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 49.17418342165371
      },
      "ko": {
       "accuracy": 0.5454545454545454,
       "cc": 55.08222770188039,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": -1.9664316484571787
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 55.08222770188039
      },
      "mixed": {
       "accuracy": 0.9090909090909091,
       "cc": 82.54649269370202,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 47.63947808110607
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 82.54649269370202
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": 9.711332583883976,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 33.33333333333333,
        "score": -3.920784049647974
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 9.711332583883976
      },
      "no": {
       "accuracy": 0.5714285714285714,
       "cc": 50.81967213114754,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 50.81967213114754
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 34.47099050276897,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 19.999999999999996,
        "score": 7.024689916702187
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 34.47099050276897
      },
      "pt": {
       "accuracy": 0.7222222222222222,
       "cc": 74.79648342566703,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": 100.0,
        "score": 60.570354799614194
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 74.79648342566703
      },
      "sv": {
       "accuracy": 0.375,
       "cc": -8.847329747252749,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 50.0,
        "score": -43.20865590842491
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 39.204683185393755,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": 10.596505696532155
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 39.204683185393755
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": -4.6691240891629535,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -100.0,
        "score": 29.47088860207636
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 52.01465673807612,
       "cc_by_type": {
        "choice": 67.56756756756756,
        "noul": 100.0,
        "score": -11.523597353339188
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 52.01465673807612
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6786666666666666,
       "cc": 58.36932112627397,
       "cc_by_type": {
        "choice": 58.36932112627397
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 58.36932112627397
      },
      "noul": {
       "accuracy": 0.7813333333333333,
       "cc": 56.266666666666666,
       "cc_by_type": {
        "noul": 56.266666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 56.266666666666666
      },
      "score": {
       "accuracy": 0.5066666666666667,
       "cc": 33.21347273527731,
       "cc_by_type": {
        "score": 33.21347273527731
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 33.21347273527731
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6666666666666666,
       "cc": 47.43548710573032,
       "cc_by_type": {
        "choice": 60.46128500823723,
        "noul": 60.00000000000001,
        "score": 21.845176308953697
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 47.43548710573032
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 41.491312000045305,
       "cc_by_type": {
        "choice": 41.08019639934533,
        "noul": 60.00000000000001,
        "score": 23.393739600790585
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 41.491312000045305
      },
      "customer_support": {
       "accuracy": 0.7,
       "cc": 48.534908440165786,
       "cc_by_type": {
        "choice": 51.61290322580645,
        "noul": 50.0,
        "score": 43.9918220946909
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 48.534908440165786
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": 7.558009820211829,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": 0.0,
        "score": -17.085006683942815
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 7.558009820211829
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6111111111111112,
       "cc": 60.831849874775266,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 22.495549624325818
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 60.831849874775266
      },
      "feature_extraction": {
       "accuracy": 0.5517241379310345,
       "cc": 33.80012553149724,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": 19.999999999999996,
        "score": 32.68242787654302
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 33.80012553149724
      },
      "financial_crime": {
       "accuracy": 0.6153846153846154,
       "cc": 42.38729210014478,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 100.0,
        "score": -7.620732395217833
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 42.38729210014478
      },
      "gaming": {
       "accuracy": 0.5172413793103449,
       "cc": 29.097480905970304,
       "cc_by_type": {
        "choice": 28.712871287128706,
        "noul": 42.85714285714286,
        "score": 15.722428573639357
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 29.097480905970304
      },
      "insurance_claims": {
       "accuracy": 0.5714285714285714,
       "cc": 59.854855365826985,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": 60.00000000000001,
        "score": 98.92964546256033
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 59.854855365826985
      },
      "knowledge_graphs": {
       "accuracy": 0.4782608695652174,
       "cc": 34.55736075618144,
       "cc_by_type": {
        "choice": 26.717557251908392,
        "noul": 60.00000000000001,
        "score": 16.954525016635912
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 34.55736075618144
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 32.637731364341626,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 25.0,
        "score": -10.826643305349105
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 32.637731364341626
      },
      "legal_compliance": {
       "accuracy": 0.3076923076923077,
       "cc": -22.12982828671353,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": -0.8157143683373169
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.8275862068965517,
       "cc": 70.67750613389478,
       "cc_by_type": {
        "choice": 81.65137614678899,
        "noul": 75.0,
        "score": 55.381142254895344
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 70.67750613389478
      },
      "model_routing": {
       "accuracy": 0.7866666666666666,
       "cc": 68.49684059479237,
       "cc_by_type": {
        "choice": 79.09407665505226,
        "noul": 86.36363636363636,
        "score": 40.03280876568852
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 68.49684059479237
      },
      "moderation_trust_safety": {
       "accuracy": 0.5625,
       "cc": 32.325588967854976,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 60.00000000000001,
        "score": -15.654812043803478
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 32.325588967854976
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 38.753237007491286,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 33.33333333333333,
        "score": 13.077131457984759
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 38.753237007491286
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": 18.588938121855957,
       "cc_by_type": {
        "choice": 24.528301886792452,
        "noul": 33.33333333333333,
        "score": -2.0948208545578995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 18.588938121855957
      },
      "scientific_discovery": {
       "accuracy": 0.375,
       "cc": 6.8446821135291955,
       "cc_by_type": {
        "choice": 28.286852589641434,
        "noul": -14.28571428571429,
        "score": 6.532908036660445
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 6.8446821135291955
      },
      "search_retrieval": {
       "accuracy": 0.8636363636363636,
       "cc": 76.70730935415533,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 100.0,
        "score": 44.44173713167841
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 76.70730935415533
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.85810602487076,
      87.75388401396593
     ],
     "I": [
      46.645926045990024,
      56.00010478358418
     ],
     "capability": [
      64.53882410044396,
      71.31542904152454
     ],
     "composite": {
      "A": [
       45.32939136901336,
       55.42668230522751
      ],
      "B": [
       42.74044451252099,
       53.75204709495266
      ],
      "C": [
       31.478744006259276,
       48.301452339916715
      ]
     },
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.58998638479137,
     "capability": 69.27093170132007,
     "intelligence": 51.951877017848766
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6786666666666666,
      "calibration": 89.06830352273374,
      "cc": 58.36932112627397,
      "cc_by_type": {
       "choice": 58.36932112627397
      },
      "cc_tiered": 63.03796283179515,
      "cc_tiered_open": 70.9521688095628,
      "cc_tiered_sealed": 55.12375685402749,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.36932112627397
     },
     "noul": {
      "accuracy": 0.7813333333333333,
      "calibration": 80.38847438136696,
      "cc": 56.266666666666666,
      "cc_by_type": {
       "noul": 56.266666666666666
      },
      "cc_tiered": 59.64083005730603,
      "cc_tiered_open": 66.16425120772948,
      "cc_tiered_sealed": 53.117408906882595,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.266666666666666
     },
     "score": {
      "accuracy": 0.5066666666666667,
      "calibration": 90.31318125027343,
      "cc": 33.21347273527731,
      "cc_by_type": {
       "score": 33.21347273527731
      },
      "cc_tiered": 33.176838164445115,
      "cc_tiered_open": 32.9560003137282,
      "cc_tiered_sealed": 33.39767601516204,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.21347273527731
     }
    },
    "raw_on_A": {
     "calibration": 86.69118099549732,
     "capability": 69.21287139307924,
     "intelligence": 51.734561790661175
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 49.795586980700435,
    "Emphasis on Accuracy 60:20:20": 47.543240666876194,
    "Emphasis on Cost 20:20:60": 45.18135214455822,
    "Emphasis on Speed 20:60:20": 58.548717831385986,
    "Intelligence only": 44.522489120413574
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Scira (zaid@scira.ai)",
   "axes": {
    "calibration": 87.80739845794042,
    "cost": 30.04968825138303,
    "intelligence": 68.17020573384414,
    "speed": 83.27058287283708
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.056815051399762694,
      "ece_score": 88.63698972004747,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.21879788452671,
      "score_before_time_limit": 88.63698972004747,
      "time_limit_failed_share": 0.016
     },
     "noul": {
      "abstention_rate": 0.15200000000000002,
      "brier": 0.06134542488590291,
      "ece": 0.061674600501855234,
      "n": 375,
      "score": 86.7299857140329,
      "score_before_time_limit": 87.66507989962895,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9866666666666667,
      "interval90_width": 0.30288888888888893,
      "n": 375,
      "nrps": 0.054088924234169763,
      "score": 89.47341177526164,
      "score_before_time_limit": 90.6825119343868,
      "time_limit_failed_share": 0.013333333333333334,
      "top_ece": 0.06613041853904722
     }
    },
    "score": 87.80739845794042
   },
   "capability": 77.98880209589228,
   "composite_ci95": {
    "A": [
     19.823210414087278,
     20.435488221185672
    ],
    "B": [
     20.365620398737217,
     21.348362906693566
    ],
    "C": [
     19.823210414087278,
     20.435488221185672
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference from the frozen 25 Sep 2026 snapshot (pricing_v15 sha256 dae86740...): qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output, x the provider-reported input tokens; free keyless endpoint, nothing generated (output 0). OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.214623 per 1,000 decisions on the common basis (USD 0.542843 over all 1,500 items) is 3.323x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.214623 per 1,000 decisions (3.323x the official cost cap USD 0.064595), against USD 0.542843 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5428432799999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.21462339393939392
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 21,
    "n_over_limit_answered_ok": 21,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Amos 1 large (Scira)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.amos.scira.ai /v1/systemone API, model `amos-1-large`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `scira` before the first request (CoS deputy #18975; every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Free, keyless; 1 s spacing for the documented 60 requests/minute.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.71062454329342,
    "I_sealed": 65.62978692439485,
    "base": 68.17020573384414,
    "excess": -4.7586891514571406,
    "gap": 5.08083761889857,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 83.93848091567699,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 84.17930125247199,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 69.17046238785369,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 39.99999999999999,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 59.02293032634961,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.3524417552081,
       "hard": 68.76756728678528,
       "judge": 55.49798398673283,
       "standard": 39.65632020047416
      }
     },
     "sealed|choice": {
      "cc": 83.05942075029978,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 77.62229675031803,
       "judge": 84.1332027424094,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 57.620837217337474,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 47.272727272727266,
       "judge": 53.846153846153854,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 56.20910280554729,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.12768431971863,
       "hard": 54.857051756337924,
       "judge": 53.445570314943325,
       "standard": 51.5992128827863
      }
     }
    }
   },
   "jevbench_score": 20.25053358459254,
   "key": "amos-1-large",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 open weights declared by the submitter (mukaddamzaid/amos-1-large); the measured object is Scira's hosted endpoint, whose deployed weights were not inspected",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.amos.scira.ai/v1/systemone#amos-1-large",
   "noul_decisive": {
    "acc_among_decisive": 0.9651898734177216,
    "decisive": 316,
    "decisive_rate": 0.8517520215633423,
    "n": 375,
    "supported": true,
    "valid": 371
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 35,
    "axes": {
     "calibration": 88.99486051802106,
     "cost": 30.04968825138303,
     "intelligence": 69.72341288828827,
     "speed": 83.61644312651299
    },
    "board_release": "v1.6.43",
    "capability": 79.35913670315466,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.000010539179282,
      20.572867919013834
     ],
     "B": [
      20.620413647103454,
      21.54072052065241
     ],
     "C": [
      20.000010539179282,
      20.572867919013834
     ]
    },
    "jevbench_score": 20.401636755316044,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 130,
    "ranks": {
     "A": 130,
     "B": 123,
     "C": 112,
     "capability": 27
    },
    "scores": {
     "A": 20.401636755316044,
     "B": 21.207017501806387,
     "C": 20.401636755316044
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.5090118609950878,
     "p50_s_raw": 0.5090118609950878,
     "p95_s_adjusted": 0.8543341861019143,
     "p95_s_raw": 0.8543341861019143
    },
    "views": {
     "Balanced 33:33:33": 18.186993256482154,
     "Emphasis on Accuracy 60:20:20": 20.460803958388272,
     "Emphasis on Cost 20:20:60": 14.317576145069761,
     "Emphasis on Speed 20:60:20": 21.628687597892217,
     "Intelligence only": 25.18364392366508
    }
   },
   "rank": 148,
   "ranked": true,
   "ranks": {
    "A": 148,
    "B": 144,
    "C": 130,
    "capability": 40
   },
   "repo": "https://huggingface.co/mukaddamzaid/amos-1-large",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.25053358459254,
    "B": 20.996167711057787,
    "C": 20.25053358459254
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.532363753503887,
    "p50_s_raw": 0.532363753503887,
    "p95_s_adjusted": 0.8845720937955772,
    "p95_s_raw": 0.8845720937955772
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 85.70884085363811,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 73.79318922758102
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 85.70884085363811
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 50.47615368112714,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 43.99999999999999,
        "score": 32.879150332883
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 50.47615368112714
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 86.67411659317217,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 70.68178970724917
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 86.67411659317217
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.2317845951076,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 89.69535378532278
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.2317845951076
      },
      "jaggedness": {
       "accuracy": 0.7222222222222222,
       "cc": 52.46040101820392,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 29.729729729729737,
        "score": 60.85647813720154
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 52.46040101820392
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.32430365063594,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 95.8762886597938,
        "score": 58.85139908175916
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.32430365063594
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 43.067969834272596,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 25.0,
        "score": 53.68844558529201
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 43.067969834272596
      },
      "multilingual": {
       "accuracy": 0.7818181818181819,
       "cc": 61.422268079528465,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 44.44444444444444,
        "score": 58.407296214095304
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 61.422268079528465
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 75.73214123542938,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 60.529757039621465
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 75.73214123542938
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 81.14468880284434,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 55.44607842054504
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.14468880284434
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 60.36687776140042,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 41.66666666666667,
        "score": 60.267299950867894
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 60.36687776140042
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 61.882864079814546,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100.0,
        "score": 23.743830334681725
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 61.882864079814546
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 52.73782751626438,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 24.880149215459824
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.73782751626438
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.90096934595631,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 75.59671334760345
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.90096934595631
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 53.05558601212194,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 92.6863111089915
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 53.05558601212194
      },
      "en": {
       "accuracy": 0.8340573414422241,
       "cc": 69.82585015619428,
       "cc_by_type": {
        "choice": 84.62308246177126,
        "noul": 66.18705035971225,
        "score": 58.6674176470993
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 69.82585015619428
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 70.72155656154153,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 21.952597906484318
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 70.72155656154153
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 74.873681863149,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 91.28771225611368
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 74.873681863149
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 81.05883440654436,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 85.52944439610368
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 81.05883440654436
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 61.66610015218894,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 50.0,
        "score": 68.51785352919251
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 61.66610015218894
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 67.97700324005393,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 17.96062780278067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.97700324005393
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 43.73893493579494,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 11.216804807384817
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 43.73893493579494
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 11.692657175340818,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 35.077971526022466
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 11.692657175340818
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 97.24507216985027,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 91.73521650955081
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 97.24507216985027
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -17.41688895535159,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": -26.276640892028812
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75.0
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 77.465615091364,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80.0,
        "score": 60.01428073748598
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 77.465615091364
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 63.573232278872894,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 62.54055717839828
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 63.573232278872894
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 56.68991453443647,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 61.13357339054346
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 56.68991453443647
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 67.58806742727757,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 52.764202281832695
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 67.58806742727757
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.91469064595238,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 75.00213645398617
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.91469064595238
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 70.9322772713171,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 79.463498480618
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 70.9322772713171
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8746666666666667,
       "cc": 83.8114009070208,
       "cc_by_type": {
        "choice": 83.8114009070208
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.8114009070208
      },
      "noul": {
       "accuracy": 0.8133333333333334,
       "cc": 62.66666666666667,
       "cc_by_type": {
        "noul": 62.66666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.66666666666667
      },
      "score": {
       "accuracy": 0.7333333333333333,
       "cc": 57.91409320050066,
       "cc_by_type": {
        "score": 57.91409320050066
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 57.91409320050066
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 12.801234460928859,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -50.0,
        "score": 36.40370338278658
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 12.801234460928859
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 83.81931093174488,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 70.09147316790545
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 83.81931093174488
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 25.116387537117735,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0.0,
        "score": 7.869487814605236
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 25.116387537117735
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -4.393060324753523,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -52.94526284560559
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.641138161384106,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 69.74950144067407
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.641138161384106
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -4.760219093872943,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -122.45675791054964
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.8,
       "cc": 54.586939841529194,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 59.27902680750074
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 54.586939841529194
      },
      "gaming": {
       "accuracy": 0.9090909090909091,
       "cc": 71.2806493361724,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50.0,
        "score": 74.29490968099105
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 71.2806493361724
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 16.02023316616262,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 30.560699498487853
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 16.02023316616262
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 67.5820202385729,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 61.77532552656804
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 67.5820202385729
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 62.07152141651769,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 42.2869156707417
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 62.07152141651769
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": 0.9993600672424611,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": -1.3029950670898183
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.9993600672424611
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 73.86169788565626,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 61.58509365696874
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 73.86169788565626
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 81.67870057339692,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 100.0,
        "score": 49.711426395515446
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.67870057339692
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 19.51262058240074,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 70.05417652647284
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 19.51262058240074
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 57.303088588281206,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 72.00211627552143
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.303088588281206
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 26.574834589572514,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -50.0,
        "score": 29.724503768717547
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 26.574834589572514
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 48.40292921906184,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 0.0,
        "score": 59.701541280373924
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 48.40292921906184
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 97.1544132796688,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 91.46323983900642
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.1544132796688
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.54175553237744,
      88.18443530250994
     ],
     "I": [
      63.229435649690274,
      72.06788091953597
     ],
     "capability": [
      74.10480215942516,
      79.41671065887567
     ],
     "composite": {
      "A": [
       19.823210414087278,
       20.435488221185672
      ],
      "B": [
       20.365620398737217,
       21.348362906693566
      ],
      "C": [
       19.823210414087278,
       20.435488221185672
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.80739845794042,
     "capability": 77.98880209589228,
     "intelligence": 68.17020573384414
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8746666666666667,
      "calibration": 87.21879788452671,
      "cc": 83.8114009070208,
      "cc_by_type": {
       "choice": 83.8114009070208
      },
      "cc_tiered": 83.49895083298838,
      "cc_tiered_open": 83.93848091567699,
      "cc_tiered_sealed": 83.05942075029978,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.8114009070208
     },
     "noul": {
      "accuracy": 0.8133333333333334,
      "calibration": 86.7299857140329,
      "cc": 62.66666666666667,
      "cc_by_type": {
       "noul": 62.66666666666667
      },
      "cc_tiered": 63.395649802595585,
      "cc_tiered_open": 69.17046238785369,
      "cc_tiered_sealed": 57.620837217337474,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.66666666666667
     },
     "score": {
      "accuracy": 0.7333333333333333,
      "calibration": 89.47341177526164,
      "cc": 57.91409320050066,
      "cc_by_type": {
       "score": 57.91409320050066
      },
      "cc_tiered": 57.61601656594845,
      "cc_tiered_open": 59.02293032634961,
      "cc_tiered_sealed": 56.20910280554729,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.91409320050066
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.016000000000000014,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.072801136912016,
    "Emphasis on Accuracy 60:20:20": 20.224788770596167,
    "Emphasis on Cost 20:20:60": 14.2749725946048,
    "Emphasis on Speed 20:60:20": 21.50613973787975,
    "Intelligence only": 24.622635586625112
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Scira (zaid@scira.ai)",
   "axes": {
    "calibration": 87.60140992244203,
    "cost": 48.74716696332005,
    "intelligence": 38.09902514689999,
    "speed": 86.2337589756876
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06666585481947378,
      "ece_score": 86.66682903610524,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 85.51127131562384,
      "score_before_time_limit": 86.66682903610524,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.43999999999999995,
      "brier": 0.12374828347808393,
      "ece": 0.06865945905447007,
      "n": 375,
      "score": 86.26810818910599,
      "score_before_time_limit": 86.26810818910599,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.4971111111111112,
      "n": 375,
      "nrps": 0.09858810096196241,
      "score": 91.02485026259626,
      "score_before_time_limit": 91.02485026259626,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04045744689305625
     }
    },
    "score": 87.60140992244203
   },
   "capability": 62.85021753467101,
   "composite_ci95": {
    "A": [
     21.482358613369826,
     41.7184973132223
    ],
    "B": [
     18.994596231491883,
     38.79119242357073
    ],
    "C": [
     14.918304592617936,
     28.971178689737705
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference from the frozen 25 Sep 2026 snapshot (pricing_v15 sha256 dae86740...): qwen/qwen3.5-9b USD 0.09999999999999999/M input, USD 0.15/M output, x the provider-reported input tokens; free keyless endpoint, nothing generated (output 0). INSIDE THE OFFICIAL COST CAP on the common basis: USD 0.051101 per 1,000 decisions is 0.791x the official cap USD 0.064595. Over all 1,500 items the row cost USD 0.129248 (2.001x the cap) and was disclosed as ranked over the cap in v1.6.29; that disclosure is superseded. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.051101 per 1,000 decisions (0.791x the official cost cap USD 0.064595), against USD 0.129248 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.12924839999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05110080808080808
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 10,
    "n_over_limit_answered_ok": 10,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Amos 1 medium (Scira)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.amos.scira.ai /v1/systemone API, model `amos-1-medium`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `scira` before the first request (CoS deputy #18975; every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Free, keyless; 1 s spacing for the documented 60 requests/minute.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 43.6624142098521,
    "I_sealed": 32.53563608394788,
    "base": 38.09902514689999,
    "excess": 1.2872513555485128,
    "gap": 11.126778125904224,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 69.12490790461862,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 22.332643202208423,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": 0.0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 39.52969152272925,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.14080159637061,
       "hard": 44.43989097468418,
       "judge": 36.67618524977716,
       "standard": 30.18399699142684
      }
     },
     "sealed|choice": {
      "cc": 62.60628592189268,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 60.969122238926786,
       "judge": 60.33300685602351,
       "standard": 62.982493144906144
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 31.034482758620683,
       "hard": -9.090909090909093,
       "judge": -16.92307692307692,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 35.00062232995097,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 49.394405829279044,
       "hard": 31.30311505659157,
       "judge": 33.35206227687498,
       "standard": 37.67158520661971
      }
     }
    }
   },
   "jevbench_score": 31.638672185087426,
   "key": "amos-1-medium",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 open weights declared by the submitter (mukaddamzaid/amos-1-medium); the measured object is Scira's hosted endpoint, whose deployed weights were not inspected",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.amos.scira.ai/v1/systemone#amos-1-medium",
   "noul_decisive": {
    "acc_among_decisive": 0.9714285714285714,
    "decisive": 210,
    "decisive_rate": 0.56,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 31,
    "axes": {
     "calibration": 87.98659582926916,
     "cost": 48.74716696332005,
     "intelligence": 38.36362733252365,
     "speed": 86.48270625779318
    },
    "board_release": "v1.6.43",
    "capability": 63.175111580896406,
    "capability_api_board_rank": 24,
    "capability_open_board_rank": null,
    "capability_rank": 69,
    "composite_ci95": {
     "A": [
      22.106438163001933,
      42.39900600898038
     ],
     "B": [
      19.591717479749285,
      39.302428856817606
     ],
     "C": [
      15.351693168751336,
      29.44375417290304
     ]
    },
    "jevbench_score": 32.20168753691601,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 96,
    "ranks": {
     "A": 96,
     "B": 98,
     "C": 98,
     "capability": 111
    },
    "scores": {
     "A": 32.20168753691601,
     "B": 29.274084226551246,
     "C": 22.362283011747234
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.44348932350840187,
     "p50_s_raw": 0.44348932350840187,
     "p95_s_adjusted": 0.5068112015054794,
     "p95_s_raw": 0.5068112015054794
    },
    "views": {
     "Balanced 33:33:33": 28.872147628956235,
     "Emphasis on Accuracy 60:20:20": 25.371533776069526,
     "Emphasis on Cost 20:20:60": 28.21250381149062,
     "Emphasis on Speed 20:60:20": 34.42715214171818,
     "Intelligence only": 21.46731530678655
    }
   },
   "rank": 110,
   "ranked": true,
   "ranks": {
    "A": 110,
    "B": 112,
    "C": 114,
    "capability": 131
   },
   "repo": "https://huggingface.co/mukaddamzaid/amos-1-medium",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 31.638672185087426,
    "B": 28.73772686231222,
    "C": 21.97130012853293
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.44756228550249944,
    "p50_s_raw": 0.44756228550249944,
    "p95_s_adjusted": 0.5318272908531071,
    "p95_s_raw": 0.5318272908531071
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 70.72841647184556,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 28.851916082203388
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 70.72841647184556
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 14.975090543066846,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 4.0000000000000036,
        "score": 4.551994852954511
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.975090543066846
      },
      "composite_gen": {
       "accuracy": 0.49504950495049505,
       "cc": 6.091925072879623,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -62.96296296296296,
        "score": 13.759063384853864
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 6.091925072879623
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 91.49611264605214,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 74.48833793815639
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 91.49611264605214
      },
      "jaggedness": {
       "accuracy": 0.5111111111111111,
       "cc": 12.733145931714104,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": -40.54054054054054,
        "score": 33.60042106908033
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 12.733145931714104
      },
      "llm_router": {
       "accuracy": 0.7827476038338658,
       "cc": 63.07665363153567,
       "cc_by_type": {
        "choice": 71.47133926212908,
        "noul": 62.88659793814433,
        "score": 54.87202369433361
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.07665363153567
      },
      "long_state": {
       "accuracy": 0.2,
       "cc": -3.9955246973262035,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0.0,
        "score": 17.045683972537518
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -13.788806870177382,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75.0,
        "score": -16.881884528057924
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6618181818181819,
       "cc": 40.822452938700465,
       "cc_by_type": {
        "choice": 70.6553635474405,
        "noul": 5.555555555555558,
        "score": 46.256439713105344
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.822452938700465
      },
      "negation_pair_gen": {
       "accuracy": 0.5,
       "cc": -6.27999295377069,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -80.0,
        "score": -12.173312194645414
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 72.02153229907263,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 100.0,
        "score": 49.09762993025094
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 72.02153229907263
      },
      "usecase": {
       "accuracy": 0.5954545454545455,
       "cc": 33.58098123168537,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": 0.0,
        "score": 37.50764957740904
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 33.58098123168537
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -5.625522909037805,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -59.73371158425627
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 49.37440980616833,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 35.52722327411174
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 49.37440980616833
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 86.26833998280311,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 74.73422348823236
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.26833998280311
      },
      "el": {
       "accuracy": 0.4,
       "cc": 14.87006399662579,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100.0,
        "score": 94.88952159881592
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 14.87006399662579
      },
      "en": {
       "accuracy": 0.629887054735013,
       "cc": 34.99976613724583,
       "cc_by_type": {
        "choice": 62.65605740715879,
        "noul": 6.474820143884896,
        "score": 35.8684208606938
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 34.99976613724583
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 62.123511307964144,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -13.629466076107576
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.123511307964144
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 36.950170300317374,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 76.97072948018709
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 36.950170300317374
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 46.162580842492474,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 80.84068370394802
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 46.162580842492474
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 15.044394910666796,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25.0,
        "score": -13.107038731686771
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 15.044394910666796
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 59.21561631194971,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -8.32353298153199
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 59.21561631194971
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 40.035094113356514,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 19.999999999999996,
        "score": 28.699404659370586
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 40.035094113356514
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -0.99009225371654,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 41.47416768329484
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.0593334229456,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 91.84466693550348
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.0593334229456
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -6.718549297485519,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 31.79240405559539
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 45.495999792566494,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 31.72287030448745
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.495999792566494
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.80789114038846,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 65.45844065746472
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.80789114038846
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 17.008028029074485,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 47.61982876807451
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 17.008028029074485
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 63.07036542482134,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 10.950226709246625
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 63.07036542482134
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.65635733546749,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 59.22713652253151
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.65635733546749
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 1.6406301152936227,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -36.32104750722647
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.6406301152936227
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7253333333333334,
       "cc": 64.52285730687538,
       "cc_by_type": {
        "choice": 64.52285730687538
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 64.52285730687538
      },
      "noul": {
       "accuracy": 0.544,
       "cc": 8.800000000000008,
       "cc_by_type": {
        "noul": 8.800000000000008
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 8.800000000000008
      },
      "score": {
       "accuracy": 0.5653333333333334,
       "cc": 36.795063482577376,
       "cc_by_type": {
        "score": 36.795063482577376
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.795063482577376
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -11.612698505906499,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -50.0,
        "score": -4.838095517719498
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 24.522153858600834,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 11.111111111111116,
        "score": -18.911109162637807
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 24.522153858600834
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 30.995651582372062,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50.0,
        "score": 8.027605153620243
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.995651582372062
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -21.304913464984477,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -100.0,
        "score": -5.435208231210731
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 19.91667593401915,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 36.56162200495601
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 19.91667593401915
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -23.035431154993848,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -85.45849472284317
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 21.961594726546508,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 38.15369174266473
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.961594726546508
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": -2.6028106451585344,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100.0,
        "score": 44.456376426893726
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 14.729500546085612,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60.0,
        "score": 45.85516830492351
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 14.729500546085612
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 18.37776298144904,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0.0,
        "score": 32.2210833768953
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 18.37776298144904
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 37.43981895623384,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -11.111111111111116,
        "score": 23.430567979812633
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 37.43981895623384
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -26.159304338357146,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -13.96178398281338
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 26.337856947168092,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 33.56251517931618
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 26.337856947168092
      },
      "model_routing": {
       "accuracy": 0.8341013824884793,
       "cc": 69.75791357046573,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 87.87878787878789,
        "score": 41.265082702739164
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.75791357046573
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 12.28249541563279,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 48.363801026168986
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 12.28249541563279
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 62.49576394692955,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 54.15395850745532
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.49576394692955
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 41.45276807709171,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 39.13662935442785
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 41.45276807709171
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 10.829423826169853,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 38.28537292778492
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 10.829423826169853
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 77.41282379352732,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 82.23847138058198
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.41282379352732
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.79575916328675,
      88.21506895592145
     ],
     "I": [
      32.48961964612892,
      43.00911209428756
     ],
     "capability": [
      58.2975259643926,
      64.52717043871775
     ],
     "composite": {
      "A": [
       21.482358613369826,
       41.7184973132223
      ],
      "B": [
       18.994596231491883,
       38.79119242357073
      ],
      "C": [
       14.918304592617936,
       28.971178689737705
      ]
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.60140992244203,
     "capability": 62.85021753467101,
     "intelligence": 38.09902514689999
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7253333333333334,
      "calibration": 85.51127131562384,
      "cc": 64.52285730687538,
      "cc_by_type": {
       "choice": 64.52285730687538
      },
      "cc_tiered": 65.86559691325564,
      "cc_tiered_open": 69.12490790461862,
      "cc_tiered_sealed": 62.60628592189268,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 64.52285730687538
     },
     "noul": {
      "accuracy": 0.544,
      "calibration": 86.26810818910599,
      "cc": 8.800000000000008,
      "cc_by_type": {
       "noul": 8.800000000000008
      },
      "cc_tiered": 11.166321601104212,
      "cc_tiered_open": 22.332643202208423,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 8.800000000000008
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 91.02485026259626,
      "cc": 36.795063482577376,
      "cc_by_type": {
       "score": 36.795063482577376
      },
      "cc_tiered": 37.26515692634011,
      "cc_tiered_open": 39.52969152272925,
      "cc_tiered_sealed": 35.00062232995097,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.795063482577376
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 28.370624106486808,
    "Emphasis on Accuracy 60:20:20": 24.892625409924076,
    "Emphasis on Cost 20:20:60": 27.764641557653366,
    "Emphasis on Speed 20:60:20": 33.836817745655395,
    "Intelligence only": 21.02617789811459
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Arcbright (arcbright on Hugging Face)",
   "axes": {
    "calibration": 86.16803253832052,
    "cost": 57.38209314881071,
    "intelligence": 43.75476217706713,
    "speed": 91.60645552542691
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06217474017608143,
      "ece_score": 87.56505196478372,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 87.56505196478372,
      "score_before_time_limit": 87.56505196478372,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3672922252010724,
      "brier": 0.11447032458100269,
      "ece": 0.07345892216986336,
      "n": 373,
      "score": 85.30821556602733,
      "score_before_time_limit": 85.30821556602733,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9516129032258065,
      "interval90_width": 0.4168906810035842,
      "n": 372,
      "nrps": 0.09282356443339915,
      "score": 85.63083008415052,
      "score_before_time_limit": 85.63083008415052,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09727991694179526
     }
    },
    "score": 86.16803253832052
   },
   "capability": 64.96139735769383,
   "composite_ci95": {
    "A": [
     34.31872379235919,
     61.97425369058897
    ],
    "B": [
     30.714936982089018,
     57.86532223869607
    ],
    "C": [
     23.832447078027215,
     43.03767617402012
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), nearest-listed-sibling reference: declared base google/gemma-4-12B-it has no market listing (UNLISTED in the frozen pricing_v15 module, sha256 6202ef78a2c83cbffc9389cf1f37df70fe0c85ed03f4dd5c5f997f3d31aae13a), priced through the frozen catalog's nearest listed sibling of the same family and size class: google/gemma-3-12b-it USD 0.05/M input, USD 0.15/M output (the fixed Gemma-4-12B precedent, same basis as the published metask_jev_rain_12b_trained row). Estimated, not charged, not a quote from the author or any host. USD per 1,000 decisions on the denominator of the release basis (the same-items cost rule for this draw prices the common item set without the 15 very long items the author's server refuses, for every row alike). Tokens as reported by the author's own server usage block (input_tokens = tokens of the rendered chat prompt incl. system prompt, options and the answer prefill; output_tokens = 1, exactly one vLLM forward pass per answered single-question row - nothing is generated, the option-code logits are read in that one pass). The offline count through the author's own render.py and the pinned tokenizer predicted the server's numbers EXACTLY, item by item: 777,809 input tokens over the 1,485 answered items, and 15 HTTP 422 refusals at the pre-registered ids and positions.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.026075466666666672
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.026338855218855223
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Arcbright Elf 1 12B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-20",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (compute capability 12.0, driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.477/h. The author's own `python3 -m elf_serve` from the pinned commit, ELF_MODEL pointed at the commit-pinned local snapshot, on 127.0.0.1:8000 inside a network-none Docker container with the cut proven from inside (loopback only, no default route, DNS and TCP fail), offline and credential-free, one request at a time. Timed over loopback on the VM. Measured 20 Oct 2026.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 49.76480591481253,
    "I_sealed": 37.74471843932173,
    "base": 43.75476217706713,
    "excess": 2.180560705135086,
    "gap": 12.020087475490797,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 69.65737475588615,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 64.40342781806197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 28.84057971014493,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 30.434782608695656,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 50.796463278406506,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.67881522863554,
       "hard": 58.047006362950924,
       "judge": 42.020712273591265,
       "standard": 40.517827641425995
      }
     },
     "sealed|choice": {
      "cc": 63.52389731980814,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 57.84665201804093,
       "judge": 61.21449559255633,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 14.103279119233829,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 41.37931034482758,
       "hard": 7.272727272727275,
       "judge": 4.615384615384621,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 35.60697887892322,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 54.33201940856638,
       "hard": 26.4903029591115,
       "judge": 36.034132628127544,
       "standard": 43.8370798299186
      }
     }
    }
   },
   "jevbench_score": 48.77406470689909,
   "key": "arcbright-elf-1-12b",
   "last_measured_on": "2026-10-20",
   "licence": "Apache-2.0 (model card, root LICENSE and serve/LICENSE; base google/gemma-4-12B-it under the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.34",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "arcbright/elf-1-12b@da29a170016f3a179821296bcc8221ac222a44c6",
   "noul_decisive": {
    "acc_among_decisive": 0.9661016949152542,
    "decisive": 236,
    "decisive_rate": 0.6327077747989276,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.16803253832052,
     "cost": 57.38209314881071,
     "intelligence": 43.75476217706713,
     "speed": 91.97994026431576
    },
    "board_release": "v1.6.43",
    "capability": 64.96139735769383,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 38,
    "capability_rank": 60,
    "composite_ci95": {
     "A": [
      34.34154919677984,
      62.01915333142578
     ],
     "B": [
      30.729584588233603,
      57.89694395385064
     ],
     "C": [
      23.848298053319336,
      43.0688564801568
     ]
    },
    "jevbench_score": 48.80851298722302,
    "measured_in": "v1.6.34",
    "method": "v1.6",
    "open_board_rank": 42,
    "rank": 66,
    "ranks": {
     "A": 66,
     "B": 68,
     "C": 72,
     "capability": 102
    },
    "scores": {
     "A": 48.80851298722302,
     "B": 44.723760281190714,
     "C": 33.894800685571546
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21816718400006038,
     "p50_s_raw": 0.034083592000030194,
     "p95_s_adjusted": 0.2905471016999911,
     "p95_s_raw": 0.07027355084999556
    },
    "views": {
     "Balanced 33:33:33": 44.91128082306797,
     "Emphasis on Accuracy 60:20:20": 39.529652968308206,
     "Emphasis on Cost 20:20:60": 44.51878166820759,
     "Emphasis on Speed 20:60:20": 52.52523519227407,
     "Intelligence only": 33.50703306610661
    }
   },
   "rank": 74,
   "ranked": true,
   "ranks": {
    "A": 74,
    "B": 77,
    "C": 83,
    "capability": 117
   },
   "repo": "https://huggingface.co/arcbright/elf-1-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 48.77406470689909,
    "B": 44.70061700513491,
    "C": 33.870878268679924
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23457195800002636,
    "p50_s_raw": 0.042285979000013185,
    "p95_s_adjusted": 0.2944952907999777,
    "p95_s_raw": 0.07224764539998886
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 53.221149190341464,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": -15.336552428975603
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 53.221149190341464
      },
      "calibration_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 34.513264583851004,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": 28.000000000000004,
        "score": 35.985180814119275
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 34.513264583851004
      },
      "composite_gen": {
       "accuracy": 0.6336633663366337,
       "cc": 32.83917641063007,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -18.518518518518523,
        "score": 46.30434043333555
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 32.83917641063007
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 79.39112055434624,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 78.17336166303872
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 79.39112055434624
      },
      "jaggedness": {
       "accuracy": 0.4888888888888889,
       "cc": 9.134700657058808,
       "cc_by_type": {
        "choice": 49.4706448508181,
        "noul": -45.945945945945944,
        "score": 23.87940306630427
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 9.134700657058808
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 68.30021817589943,
       "cc_by_type": {
        "choice": 75.43365325350004,
        "noul": 77.31958762886597,
        "score": 52.14741364533227
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.30021817589943
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 9.266765553937715,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 0.0,
        "score": -22.715167255712633
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 9.266765553937715
      },
      "multilingual": {
       "accuracy": 0.6654545454545454,
       "cc": 44.16630479379578,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 13.888888888888884,
        "score": 52.84543468715122
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 44.16630479379578
      },
      "negation_pair_gen": {
       "accuracy": 0.5,
       "cc": 4.563370710907104,
       "cc_by_type": {
        "choice": 63.333333333333336,
        "noul": -53.333333333333336,
        "score": 3.6901121327213127
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 4.563370710907104
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 72.7892309873725,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100.0,
        "score": 42.391716986141546
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 72.7892309873725
      },
      "usecase": {
       "accuracy": 0.5909090909090909,
       "cc": 37.411840669893564,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": 12.5,
        "score": 37.725718088112046
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 37.411840669893564
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": -2.2963023047605,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 0.0,
        "score": -68.7936688190434
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 21.105620677411725,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 17.387522554508593
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 21.105620677411725
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 78.60884686810552,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100.0,
        "score": 75.64954945387412
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 78.60884686810552
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 33.20380808987042,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 99.79764400890363
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 33.20380808987042
      },
      "en": {
       "accuracy": 0.6550825369244135,
       "cc": 41.21871964823898,
       "cc_by_type": {
        "choice": 65.5117706642584,
        "noul": 19.424460431654666,
        "score": 38.71992784880387
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 41.21871964823898
      },
      "es": {
       "accuracy": 0.6774193548387096,
       "cc": 52.57126625614512,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 75.0,
        "score": 2.289655212154784
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 52.57126625614512
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.16848435692223,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 97.74589022923665
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.16848435692223
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 76.73155887991228,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 72.54761781620745
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 76.73155887991228
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 41.739169283908474,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 50.0,
        "score": -8.022715611961729
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 41.739169283908474
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 65.060016125563,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 23.23928454192673
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 65.060016125563
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 30.419424479507466,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 28.446518077124473
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 30.419424479507466
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 8.69703083755734,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 26.09109251267202
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 8.69703083755734
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.65244468093822,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.62400070948131
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.65244468093822
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 2.0314007863726835,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": -8.62441235949667
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 2.0314007863726835
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 56.852101069094736,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": 39.99999999999999,
        "score": 61.02604506086019
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.852101069094736
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 60.2311405918991,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 63.121235564736764
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 60.2311405918991
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 7.854063750825509,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 20.157935933327575
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 7.854063750825509
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 8.62512772013944,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": -30.646355970016458
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 8.62512772013944
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 68.40627899393645,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 69.73496601406745
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 68.40627899393645
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 38.58243816800493,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 49.080647837348124
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 38.58243816800493
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7333333333333333,
       "cc": 65.55617214259745,
       "cc_by_type": {
        "choice": 65.55617214259745
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.55617214259745
      },
      "noul": {
       "accuracy": 0.608,
       "cc": 21.599999999999998,
       "cc_by_type": {
        "noul": 21.599999999999998
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 21.599999999999998
      },
      "score": {
       "accuracy": 0.56,
       "cc": 39.95424165202871,
       "cc_by_type": {
        "score": 39.95424165202871
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 39.95424165202871
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -15.780619190966505,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -75.0,
        "score": -8.341857572899514
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 43.202215257321164,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 55.55555555555556,
        "score": 11.31817096174953
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 43.202215257321164
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 26.38194646652667,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0.0,
        "score": 11.66616460283204
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 26.38194646652667
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -35.273927478672455,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -35.64634383952614
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 30.238339740416034,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 53.033859800958226
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 30.238339740416034
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -23.83827556753779,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -87.86702796047499
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 40.969829317719324,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 38.03553837332603
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 40.969829317719324
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 0.870256604504983,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -100.0,
        "score": 44.42261650341042
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.870256604504983
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 11.172183319917124,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 16.01654995975137
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.172183319917124
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 39.61960806069081,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 16.431986209195905
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 39.61960806069081
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 34.1684274583135,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 11.111111111111116,
        "score": 14.649985217317763
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 34.1684274583135
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -33.228079786697315,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -35.168110327833865
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 66.36226749196166,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 50.6031172551556
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 66.36226749196166
      },
      "model_routing": {
       "accuracy": 0.8202764976958525,
       "cc": 70.15069627663628,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 90.90909090909092,
        "score": 39.413127790947776
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 70.15069627663628
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 10.565354791083555,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 66.24500871106257
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 10.565354791083555
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 48.01941397788096,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 44.151092444320696
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 48.01941397788096
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 28.233025996560656,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0.0,
        "score": 29.0340533591401
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 28.233025996560656
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 5.217071352811868,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -33.333333333333336,
        "score": 35.94106913089937
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 5.217071352811868
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 95.3686978433633,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 96.7822501137198
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 95.3686978433633
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.62303224336958,
      87.64236723127183
     ],
     "I": [
      37.79040779205359,
      48.44381950020701
     ],
     "capability": [
      60.23623630844356,
      67.39815359481638
     ],
     "composite": {
      "A": [
       34.31872379235919,
       61.97425369058897
      ],
      "B": [
       30.714936982089018,
       57.86532223869607
      ],
      "C": [
       23.832447078027215,
       43.03767617402012
      ]
     },
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.16803253832052,
     "capability": 64.96139735769383,
     "intelligence": 43.75476217706713
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 87.56505196478372,
      "cc": 65.55617214259745,
      "cc_by_type": {
       "choice": 65.55617214259745
      },
      "cc_tiered": 66.59063603784715,
      "cc_tiered_open": 69.65737475588615,
      "cc_tiered_sealed": 63.52389731980814,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.55617214259745
     },
     "noul": {
      "accuracy": 0.608,
      "calibration": 85.30821556602733,
      "cc": 21.599999999999998,
      "cc_by_type": {
       "noul": 21.599999999999998
      },
      "cc_tiered": 21.471929414689377,
      "cc_tiered_open": 28.84057971014493,
      "cc_tiered_sealed": 14.103279119233829,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 21.599999999999998
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 85.63083008415052,
      "cc": 39.95424165202871,
      "cc_by_type": {
       "score": 39.95424165202871
      },
      "cc_tiered": 43.20172107866486,
      "cc_tiered_open": 50.796463278406506,
      "cc_tiered_sealed": 35.60697887892322,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.95424165202871
     }
    },
    "raw_on_A": {
     "calibration": 84.45522192019058,
     "capability": 62.52355684051309,
     "intelligence": 40.591891760835594
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 44.87239811742327,
    "Emphasis on Accuracy 60:20:20": 39.51157205690974,
    "Emphasis on Cost 20:20:60": 44.49584999284475,
    "Emphasis on Speed 20:60:20": 52.42959536714219,
    "Intelligence only": 33.50703306610661
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "IBOU-SEARCH",
   "axes": {
    "calibration": 82.22627427476863,
    "cost": 60.69086683660791,
    "intelligence": 27.956374646954437,
    "speed": 89.52285293002008
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13190589669297953,
      "ece_score": 73.6188206614041,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 73.6188206614041,
      "score_before_time_limit": 73.6188206614041,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.581769436997319,
      "brier": 0.1619943265239912,
      "ece": 0.0674130307496136,
      "n": 373,
      "score": 86.51739385007728,
      "score_before_time_limit": 86.51739385007728,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9919354838709677,
      "interval90_width": 0.7842741935483872,
      "n": 372,
      "nrps": 0.1214577533167964,
      "score": 86.54260831282451,
      "score_before_time_limit": 86.54260831282451,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07384504021335675
     }
    },
    "score": 82.22627427476863
   },
   "capability": 55.09132446086153,
   "composite_ci95": {
    "A": [
     12.967526399444894,
     20.808288311198115
    ],
    "B": [
     10.815260670164944,
     17.964237732170922
    ],
    "C": [
     9.005226666281176,
     14.4502002161098
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835, the module this row was first priced with (the later re-freezes only add hosted-API launch-list keys), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). Tokens: the server reports prompt_tokens 0 (not a measurement), so input is counted OFFLINE with the author's tokenizer and shared-prefix method (the method of the earlier measurement, measurement receipts on file; 1,011,369 over 1,485), output 0 under the 9 Oct 2026 rule (forward passes only). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.020432 per 1,000 decisions (0.316x the official cost cap USD 0.064595), against USD 0.020227 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.02022738
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02043169696969697
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ArseneLupin v1.1 (IBOU-SEARCH, Qwen3.5-4B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20150, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's `python -m arsenelupin serve` (engine shared_prefix, bf16, calibration T 2.45) behind the wire-name proxy.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 33.63845928917361,
    "I_sealed": 22.274290004735263,
    "base": 27.956374646954437,
    "excess": 1.524642514082636,
    "gap": 11.364169284438347,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 66.71937273976232,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965713,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": -40.0,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 34.196005127758525,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.46334077824246,
       "hard": 22.992987523752227,
       "judge": 43.70520186247461,
       "standard": 41.20457740845502
      }
     },
     "sealed|choice": {
      "cc": 47.96190215319236,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 56.16264294790343,
       "hard": 44.836359431016525,
       "judge": 48.87365328109698,
       "standard": 48.74499050833158
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 10.344827586206895,
       "hard": -30.909090909090907,
       "judge": -53.84615384615385,
       "standard": -22.388059701492537
      }
     },
     "sealed|score": {
      "cc": 18.860967861013435,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 32.91947880531089,
       "hard": 16.11073859508191,
       "judge": 16.587666801933665,
       "standard": 20.742122509347404
      }
     }
    }
   },
   "jevbench_score": 16.545556287392994,
   "key": "arsenelupin-v1.1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "IBOU-SEARCH/ArseneLupin-v1.1@411a6aefab87a4c155372c2a0a20777c2f5e1317",
   "noul_decisive": {
    "acc_among_decisive": 0.9423076923076923,
    "decisive": 156,
    "decisive_rate": 0.41823056300268097,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.22627427476863,
     "cost": 60.69086683660791,
     "intelligence": 27.956374646954437,
     "speed": 89.69506927060962
    },
    "board_release": "v1.6.43",
    "capability": 55.09132446086153,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 82,
    "capability_rank": 108,
    "composite_ci95": {
     "A": [
      12.971040788669306,
      20.8144338578416
     ],
     "B": [
      10.817216180981147,
      17.96790173408304
     ],
     "C": [
      9.007667214353683,
      14.454467956834444
     ]
    },
    "jevbench_score": 16.550252807567574,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 114,
    "rank": 151,
    "ranks": {
     "A": 151,
     "B": 152,
     "C": 151,
     "capability": 163
    },
    "scores": {
     "A": 16.550252807567574,
     "B": 14.040724977478481,
     "C": 11.493231116366374
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3189419380272739,
     "p50_s_raw": 0.08447096901363693,
     "p95_s_adjusted": 0.3363422646856634,
     "p95_s_raw": 0.0931711323428317
    },
    "views": {
     "Balanced 33:33:33": 14.793881978653419,
     "Emphasis on Accuracy 60:20:20": 11.584152166235047,
     "Emphasis on Cost 20:20:60": 16.223374091593428,
     "Emphasis on Speed 20:60:20": 18.240772757738373,
     "Intelligence only": 8.739821181204116
    }
   },
   "rank": 170,
   "ranked": true,
   "ranks": {
    "A": 170,
    "B": 171,
    "C": 171,
    "capability": 182
   },
   "repo": "https://huggingface.co/IBOU-SEARCH/ArseneLupin-v1.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.545556287392994,
    "B": 14.03802054850367,
    "C": 11.489969644022915
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3199281820037868,
    "p50_s_raw": 0.0849640910018934,
    "p95_s_adjusted": 0.34886886222520835,
    "p95_s_raw": 0.09943443111260418
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 50.51872953772545,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 100.0,
        "score": 1.5561886131763458
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 50.51872953772545
      },
      "calibration_gen": {
       "accuracy": 0.38613861386138615,
       "cc": -1.2802984282284164,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -28.000000000000004,
        "score": 3.692508745007217
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.504950495049505,
       "cc": 9.769891325776209,
       "cc_by_type": {
        "choice": 57.72357723577236,
        "noul": -62.96296296296296,
        "score": 34.54905970451922
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 9.769891325776209
      },
      "fairness": {
       "accuracy": 0.6875,
       "cc": 54.72513077918578,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 44.17539233755736
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 54.72513077918578
      },
      "jaggedness": {
       "accuracy": 0.3055555555555556,
       "cc": -19.677183728618825,
       "cc_by_type": {
        "choice": 13.378248315688179,
        "noul": -67.56756756756756,
        "score": -4.842231933977081
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6932907348242812,
       "cc": 46.00272325098312,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 34.020618556701024,
        "score": 34.89360032894183
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 46.00272325098312
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -12.636857704693645,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -75.0,
        "score": 23.687365030248962
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5563636363636364,
       "cc": 19.72366546133326,
       "cc_by_type": {
        "choice": 56.9611998695794,
        "noul": -36.111111111111114,
        "score": 38.3209076255315
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 19.72366546133326
      },
      "negation_pair_gen": {
       "accuracy": 0.36046511627906974,
       "cc": -11.788936369121076,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -73.33333333333334,
        "score": 4.633190892636774
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 63.02665837969059,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 60.00000000000001,
        "score": 53.10399916309577
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 63.02665837969059
      },
      "usecase": {
       "accuracy": 0.45,
       "cc": 8.571271059852021,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": -45.833333333333336,
        "score": 20.566754356026628
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 8.571271059852021
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 13.512897780745016,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 50.0,
        "score": 4.824407627949334
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 13.512897780745016
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 4.233223448197042,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 12.69967034459113
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 4.233223448197042
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 39.83897361366874,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 0.0,
        "score": 43.41072615074075
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 39.83897361366874
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 4.953090501411556,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -100.0,
        "score": 81.89837764948606
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 4.953090501411556
      },
      "en": {
       "accuracy": 0.5178105994787141,
       "cc": 15.967276196171342,
       "cc_by_type": {
        "choice": 50.3545233765758,
        "noul": -20.863309352517987,
        "score": 18.410614564456218
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 15.967276196171342
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 61.117636296678036,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75.0,
        "score": 18.140837111893816
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 61.117636296678036
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 2.2535777645684334,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 38.45472236474355
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 2.2535777645684334
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 29.96031603152154,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0.0,
        "score": 53.410359859270486
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 29.96031603152154
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 20.903606766234788,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -75.0,
        "score": 54.470596835017204
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 20.903606766234788
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 43.96472119872376,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 33.33333333333333,
        "score": 12.590448345456796
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 43.96472119872376
      },
      "ja": {
       "accuracy": 0.44,
       "cc": 7.47713527194653,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": 0.0,
        "score": 8.21377277374269
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 7.47713527194653
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -10.20949871018135,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 13.815948313900407
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 93.5496427429219,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 80.64892822876573
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 93.5496427429219
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -13.055905252050016,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100.0,
        "score": 38.75436216592787
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 19.62379626604363,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40.0,
        "score": 29.34113065170686
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 19.62379626604363
      },
      "pt": {
       "accuracy": 0.5714285714285714,
       "cc": 22.01660632547086,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 39.08458621271193
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 22.01660632547086
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -7.000724261111401,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60.0,
        "score": 41.12548679113388
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 3.4888991168227768,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -50.0,
        "score": 25.68408865481615
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 3.4888991168227768
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.92575718462467,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 51.777271553874016
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.92575718462467
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 9.15759259136832,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -13.77016007900238
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.15759259136832
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6253333333333333,
       "cc": 51.60642186034941,
       "cc_by_type": {
        "choice": 51.60642186034941
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 51.60642186034941
      },
      "noul": {
       "accuracy": 0.392,
       "cc": -21.599999999999998,
       "cc_by_type": {
        "noul": -21.599999999999998
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.4693333333333333,
       "cc": 22.200266391018797,
       "cc_by_type": {
        "score": 22.200266391018797
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 22.200266391018797
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -15.830990825213652,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": -24.49297247564095
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 1.7168943804986962,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -33.333333333333336,
        "score": 13.018177965512656
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.7168943804986962
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 28.7536967232783,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0.0,
        "score": 18.78141537308693
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 28.7536967232783
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -25.640628784241386,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": 28.34127154201269
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 11.906917556526437,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 27.025100495666265
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 11.906917556526437
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -28.821393224240854,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -100.0,
        "score": -10.99248155951502
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 20.561134748485653,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 33.95231180848217
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 20.561134748485653
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -17.75792051210721,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 19.896970170995452
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": 5.325356796383858,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 38.47607038915157
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 5.325356796383858
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -12.492206719922391,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -33.333333333333336,
        "score": -1.3595609163695954
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 20.266776521258546,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 5.761569873853178
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 20.266776521258546
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -20.09225148226946,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 4.239374585449685
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 5.74928641158261,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60.0,
        "score": 23.313118351830365
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 5.74928641158261
      },
      "model_routing": {
       "accuracy": 0.7004608294930875,
       "cc": 45.029199454317904,
       "cc_by_type": {
        "choice": 76.62337662337664,
        "noul": 39.393939393939405,
        "score": 19.07028234563767
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 45.029199454317904
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 1.9206201017423343,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 51.82711942230953
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 1.9206201017423343
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 20.15666397470658,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 27.229509101464224
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 20.15666397470658
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 11.531424940662966,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 14.150925068294306
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 11.531424940662966
      },
      "scientific_discovery": {
       "accuracy": 0.30434782608695654,
       "cc": -17.435804548794813,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -100.0,
        "score": 20.15635446955759
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.6666666666666666,
       "cc": 15.043192447509194,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": -100.0,
        "score": 66.48189050978736
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 15.043192447509194
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.84964659068773,
      84.32731575272373
     ],
     "I": [
      25.32894837642683,
      30.711526629669294
     ],
     "capability": [
      52.11423594286504,
      56.58071014414595
     ],
     "composite": {
      "A": [
       12.967526399444894,
       20.808288311198115
      ],
      "B": [
       10.815260670164944,
       17.964237732170922
      ],
      "C": [
       9.005226666281176,
       14.4502002161098
      ]
     },
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.22627427476863,
     "capability": 55.09132446086153,
     "intelligence": 27.956374646954437
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6253333333333333,
      "calibration": 73.6188206614041,
      "cc": 51.60642186034941,
      "cc_by_type": {
       "choice": 51.60642186034941
      },
      "cc_tiered": 57.340637446477345,
      "cc_tiered_open": 66.71937273976232,
      "cc_tiered_sealed": 47.96190215319236,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.60642186034941
     },
     "noul": {
      "accuracy": 0.392,
      "calibration": 86.51739385007728,
      "cc": -21.599999999999998,
      "cc_by_type": {
       "noul": -21.599999999999998
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.4693333333333333,
      "calibration": 86.54260831282451,
      "cc": 22.200266391018797,
      "cc_by_type": {
       "score": 22.200266391018797
      },
      "cc_tiered": 26.52848649438598,
      "cc_tiered_open": 34.196005127758525,
      "cc_tiered_sealed": 18.860967861013435,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.200266391018797
     }
    },
    "raw_on_A": {
     "calibration": 75.10885434845505,
     "capability": 52.09577580877465,
     "intelligence": 29.082697269094247
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.788878796407928,
    "Emphasis on Accuracy 60:20:20": 11.582311226133971,
    "Emphasis on Cost 20:20:60": 16.21976360383656,
    "Emphasis on Speed 20:60:20": 18.22708721790156,
    "Intelligence only": 8.739821181204116
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "attest",
   "axes": {
    "calibration": 73.84793106824938,
    "cost": 50.72941039177032,
    "intelligence": 65.23849472031306,
    "speed": 96.47488998719595
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08913950574898584,
      "ece_score": 82.17209885020283,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 82.17209885020283,
      "score_before_time_limit": 82.17209885020283,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.02680965147453085,
      "brier": 0.12374104570386348,
      "ece": 0.1233374127983535,
      "n": 373,
      "score": 75.3325174403293,
      "score_before_time_limit": 75.3325174403293,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.7016129032258065,
      "interval90_width": 0.0378584229390681,
      "n": 372,
      "nrps": 0.10002561109774619,
      "score": 64.03917691421601,
      "score_before_time_limit": 64.03917691421601,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.3095954253089667
     }
    },
    "score": 73.84793106824938
   },
   "capability": 69.54321289428123,
   "composite_ci95": {
    "A": [
     65.6462584019398,
     69.5084228209243
    ],
    "B": [
     64.41799110254772,
     69.60746615454364
    ],
    "C": [
     64.92305184156147,
     69.5084228209243
    ]
   },
   "cost": {
    "basis": "ESTIMATE, v1.5-M2 base-model reference rule: vendor-hosted API, customer list price USD 0.048/M input and USD 0.00/M output effective 2026-10-11 has been in force fewer than 30 days. Base model google/gemma-4-31b-it reference on OpenRouter USD 0.09/M input / USD 0.34/M output, snapshot fetched 2026-10-12T17:55:05Z (SHA-256 81eae2f3f94bf299c8d4499260f842c357b7a869ce82dfa1741ad6a89cf57014), same base-model estimate precedent as metask_jev_rain_31b_trained. Applied to measured response usage; failed HTTP 500 usage/billing may be unreported. No output generation; reported output tokens 0. Estimated, not a vendor bill. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.043889 per 1,000 decisions (0.679x the official cost cap USD 0.064595), against USD 0.043450 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.043449839999999997
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04388872727272727
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "attest-31B-v1.0",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane order 07d13315. Vendor-hosted attest.engineering /v1/systemone API; latency measured from Sandy (Germany), using the official per-item latency subset without API adjustment. Full S1200+P300 draw, not API A300 supplement; no API equating. 15 of 1,500 requests returned HTTP 500 after 8 retry invocations and are scored as wrong in the full denominator. Public model documentation declares frozen google/gemma-4-31B-it @842da3794eaa0b77d5f08bae87a17459d91ff475, unmodified bf16, no adapter or quantization; single forward pass reading option-label logits, no generated tokens, one fitted temperature (value not documented). Hosting implementation and deployed weights were not independently inspected. Hosted API access is not an open-weights release by the vendor; base weights are under the Gemma 4 licence. ESTIMATE, v1.5-M2 base-model reference rule: vendor-hosted API, customer list price USD 0.048/M input and USD 0.00/M output effective 2026-10-11 has been in force fewer than 30 days. Base model google/gemma-4-31b-it reference on OpenRouter USD 0.09/M input / USD 0.34/M output, snapshot fetched 2026-10-12T17:55:05Z (SHA-256 81eae2f3f94bf299c8d4499260f842c357b7a869ce82dfa1741ad6a89cf57014), same base-model estimate precedent as metask_jev_rain_31b_trained. Applied to measured response usage; failed HTTP 500 usage/billing may be unreported. No output generation; reported output tokens 0. Estimated, not a vendor bill.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 64.92424666524875,
    "I_sealed": 65.55274277537737,
    "base": 65.23849472031306,
    "excess": -10.468022880484334,
    "gap": -0.6284961101286228,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 73.17162102453656,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 68.67632850241546,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 52.92479046879421,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.941722778327325,
       "hard": 60.749677220164536,
       "judge": 51.85551193839042,
       "standard": 33.370468606892665
      }
     },
     "sealed|choice": {
      "cc": 74.62818230752828,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 71.89776801202729,
       "judge": 74.43682664054849,
       "standard": 77.2199957814807
      }
     },
     "sealed|noul": {
      "cc": 68.21259029184918,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 58.18181818181818,
       "judge": 72.30769230769232,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 53.817455726754645,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 78.07383467048999,
       "hard": 52.90470314143051,
       "judge": 47.46936317941805,
       "standard": 53.03691024654009
      }
     }
    }
   },
   "jevbench_score": 67.85694940132115,
   "key": "attest-31b-v1.0",
   "last_measured_on": "2026-10-12",
   "licence": "Hosted API; declared base weights under the Gemma 4 licence (no vendor weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.13",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475",
   "noul_decisive": {
    "acc_among_decisive": 0.8787878787878788,
    "decisive": 363,
    "decisive_rate": 0.9731903485254692,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 20,
    "axes": {
     "calibration": 73.84793106824938,
     "cost": 50.72941039177032,
     "intelligence": 65.23849472031306,
     "speed": 97.10993413996468
    },
    "board_release": "v1.6.43",
    "capability": 69.54321289428123,
    "capability_api_board_rank": 20,
    "capability_open_board_rank": null,
    "capability_rank": 49,
    "composite_ci95": {
     "A": [
      65.7193671470225,
      69.5903923988902
     ],
     "B": [
      64.47429646451545,
      69.67321341053939
     ],
     "C": [
      64.99583707590963,
      69.5903923988902
     ]
    },
    "jevbench_score": 67.9350679784662,
    "measured_in": "v1.6.13",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 38,
    "ranks": {
     "A": 38,
     "B": 33,
     "C": 32,
     "capability": 82
    },
    "scores": {
     "A": 67.9350679784662,
     "B": 67.37806588505295,
     "C": 67.9350679784662
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.10787388649987406,
     "p50_s_raw": 0.10787388649987406,
     "p95_s_adjusted": 0.18033925039962923,
     "p95_s_raw": 0.18033925039962923
    },
    "views": {
     "Balanced 33:33:33": 66.16905786253606,
     "Emphasis on Accuracy 60:20:20": 65.79366502170826,
     "Emphasis on Cost 20:20:60": 58.987811289065355,
     "Emphasis on Speed 20:60:20": 75.8338379652306,
     "Intelligence only": 65.23849472031306
    }
   },
   "rank": 43,
   "ranked": true,
   "ranks": {
    "A": 43,
    "B": 35,
    "C": 35,
    "capability": 96
   },
   "repo": "https://attest.engineering/model",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.85694940132115,
    "B": 67.31657701246269,
    "C": 67.85694940132115
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.11031594549967849,
    "p50_s_raw": 0.11031594549967849,
    "p95_s_adjusted": 0.20411396054878422,
    "p95_s_raw": 0.20411396054878422
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 91.66666666649981,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 99.99999999949942
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 91.66666666649981
      },
      "calibration_gen": {
       "accuracy": 0.7227722772277227,
       "cc": 65.68205659085977,
       "cc_by_type": {
        "choice": 61.82396606574761,
        "noul": 76.0,
        "score": 59.22220370683169
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 65.68205659085977
      },
      "composite_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 71.7068789691002,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 62.962962962962955,
        "score": 58.66173898498804
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 71.7068789691002
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 90.47619047605701,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": 100.0,
        "score": 99.99999999959958
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 90.47619047605701
      },
      "jaggedness": {
       "accuracy": 0.5722222222222222,
       "cc": 41.84478952801769,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": 56.75675675675676,
        "score": 23.63805456069382
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 41.84478952801769
      },
      "llm_router": {
       "accuracy": 0.8977635782747604,
       "cc": 83.50801061677168,
       "cc_by_type": {
        "choice": 88.90552082416131,
        "noul": 93.81443298969072,
        "score": 67.804078036463
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.50801061677168
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 60.78261745433469,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 75.0,
        "score": 56.83238844547831
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 60.78261745433469
      },
      "multilingual": {
       "accuracy": 0.7563636363636363,
       "cc": 63.88819552264045,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 61.111111111111114,
        "score": 58.91995736095108
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.88819552264045
      },
      "negation_pair_gen": {
       "accuracy": 0.7441860465116279,
       "cc": 41.72809466728673,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 6.666666666666665,
        "score": 25.184284001860192
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 41.72809466728673
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 85.01995309361071,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100.0,
        "score": 73.07787729885015
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.01995309361071
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 69.59874631177811,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 75.0,
        "score": 57.080552660824544
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 69.59874631177811
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 25.657000371915586,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -15.886141741396088
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 25.657000371915586
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 1.0,
       "cc": 99.83499124305172,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.50497372915514
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 99.83499124305172
      },
      "de": {
       "accuracy": 0.875,
       "cc": 88.7927420089141,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 90.27203133647683
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.7927420089141
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.60459342006904,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.9999999994995
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.60459342006904
      },
      "en": {
       "accuracy": 0.7793223284100782,
       "cc": 66.85046932152197,
       "cc_by_type": {
        "choice": 75.39693193883403,
        "noul": 70.50359712230217,
        "score": 54.650878903429714
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 66.85046932152197
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.69113187317208,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 19.649252063235643
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.69113187317208
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.84881602901042,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.99999999959955
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.84881602901042
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 75.35603715160619,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 68.42105263128916
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.35603715160619
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 81.9134078211331,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100.0,
        "score": 62.499999999712166
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 81.9134078211331
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 53.22313405090798,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -26.300979764657196
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 53.22313405090798
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 45.07973339782123,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": 3.833322512764714
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.07973339782123
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 46.296506956290735,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 83.33396531331667
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 46.296506956290735
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.77777777761094,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.9999999994995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.77777777761094
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 43.862463409737074,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 50.202108843929864
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 43.862463409737074
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 58.11281326204074,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 49.5733107129102
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 58.11281326204074
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 73.9630456140903,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 64.3169506313104
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.9630456140903
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 52.97872340417189,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 49.99999999974971
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 52.97872340417189
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 88.24218637816098,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 86.46568956926559
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 88.24218637816098
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.53664033508191,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 49.86798552137475
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.53664033508191
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 66.66666666666664,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -4.440892098500626e-14
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 66.66666666666664
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.804,
       "cc": 74.68378652480914,
       "cc_by_type": {
        "choice": 74.68378652480914
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 74.68378652480914
      },
      "noul": {
       "accuracy": 0.8506666666666667,
       "cc": 70.13333333333334,
       "cc_by_type": {
        "noul": 70.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 70.13333333333334
      },
      "score": {
       "accuracy": 0.6506666666666666,
       "cc": 55.14020270807265,
       "cc_by_type": {
        "score": 55.14020270807265
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 55.14020270807265
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 41.05013190841991,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 0.0,
        "score": 39.15039572525971
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 41.05013190841991
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 69.18221735790132,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 7.546652073703974
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 69.18221735790132
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 33.73983739844072,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": -49.999999999799805
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 33.73983739844072
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 40.20929030056567,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": 100.0,
        "score": 25.891028796433858
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 40.20929030056567
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.627914855635204,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 74.53591847994909
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.627914855635204
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 81.67133102077554,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100.0,
        "score": 95.32845847113167
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 81.67133102077554
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 51.05695837350043,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 39.72549696924077
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.05695837350043
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 31.06744966418366,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -50.0,
        "score": 85.01419568244644
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 31.06744966418366
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 45.83333333332221,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100.0,
        "score": -3.3373304120232206e-11
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 45.83333333332221
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 79.31347738618848,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100.0,
        "score": 63.636363636081526
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 79.31347738618848
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 72.4290843339358,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 51.13738220077384
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 72.4290843339358
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 74.35692248903656,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 87.58689649936777
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 74.35692248903656
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 78.81343307279933,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 76.44029921839794
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 78.81343307279933
      },
      "model_routing": {
       "accuracy": 0.8894009216589862,
       "cc": 82.42648273169006,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 100.0,
        "score": 56.63009754571953
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.42648273169006
      },
      "moderation_trust_safety": {
       "accuracy": 0.9444444444444444,
       "cc": 92.05979877042455,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 76.17939631127363
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 92.05979877042455
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 57.779434740179134,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.43115473121522
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.779434740179134
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 47.99092050610208,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50.0,
        "score": 23.52941176461164
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 47.99092050610208
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 59.19668873382481,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 73.24224011451793
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 59.19668873382481
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.99999999986125,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.99999999958375
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.99999999986125
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      69.51167624462902,
      77.41120372530025
     ],
     "I": [
      59.64991031101489,
      70.14236746435076
     ],
     "capability": [
      65.30829911391156,
      72.94151936366131
     ],
     "composite": {
      "A": [
       65.6462584019398,
       69.5084228209243
      ],
      "B": [
       64.41799110254772,
       69.60746615454364
      ],
      "C": [
       64.92305184156147,
       69.5084228209243
      ]
     },
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.84793106824938,
     "capability": 69.54321289428123,
     "intelligence": 65.23849472031306
    },
    "per_type": {
     "choice": {
      "accuracy": 0.804,
      "calibration": 82.17209885020283,
      "cc": 74.68378652480914,
      "cc_by_type": {
       "choice": 74.68378652480914
      },
      "cc_tiered": 73.89990166603242,
      "cc_tiered_open": 73.17162102453656,
      "cc_tiered_sealed": 74.62818230752828,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.68378652480914
     },
     "noul": {
      "accuracy": 0.8506666666666667,
      "calibration": 75.3325174403293,
      "cc": 70.13333333333334,
      "cc_by_type": {
       "noul": 70.13333333333334
      },
      "cc_tiered": 68.44445939713232,
      "cc_tiered_open": 68.67632850241546,
      "cc_tiered_sealed": 68.21259029184918,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 70.13333333333334
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 64.03917691421601,
      "cc": 55.14020270807265,
      "cc_by_type": {
       "score": 55.14020270807265
      },
      "cc_tiered": 53.37112309777443,
      "cc_tiered_open": 52.92479046879421,
      "cc_tiered_sealed": 53.817455726754645,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.14020270807265
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.07027859433674,
    "Emphasis on Accuracy 60:20:20": 65.7350327220277,
    "Emphasis on Cost 20:20:60": 58.94067738965652,
    "Emphasis on Speed 20:60:20": 75.60067159595273,
    "Intelligence only": 65.23849472031306
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Remek Kinas (Remigiusz Kinas; Hugging Face Remek, GitHub rkinas)",
   "axes": {
    "calibration": 90.20661225231586,
    "cost": 50.37412598110784,
    "intelligence": 74.05810874808022,
    "speed": 86.63193265367512
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.042664316931708864,
      "ece_score": 91.46713661365823,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.24757479214279,
      "score_before_time_limit": 91.46713661365823,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.07774798927613946,
      "brier": 0.06996825861082419,
      "ece": 0.052891090232176925,
      "n": 373,
      "score": 88.94486578314562,
      "score_before_time_limit": 89.42178195356462,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9596774193548387,
      "interval90_width": 0.196236559139785,
      "n": 372,
      "nrps": 0.04515355626268863,
      "score": 91.42739618165918,
      "score_before_time_limit": 92.16471389280159,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.05577608294063997
     }
    },
    "score": 90.20661225231586
   },
   "capability": 82.13236050019805,
   "composite_ci95": {
    "A": [
     69.8943752475979,
     72.42598956613317
    ],
    "B": [
     69.75445190953462,
     73.46518541564177
    ],
    "C": [
     69.8943752475979,
     72.42598956613317
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base google/gemma-4-31B-it -> google/gemma-4-31b-it USD 0.09/M input, USD 0.33999999999999997/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. USD per 1,000 decisions over the common 1,485-item cost basis. Tokens as reported by the author's own server (usage.input_tokens = tokens of the rendered chat prompt incl. system prompt, options and the answer prefill; output_tokens 0, nothing is generated - the model reads the option-code logits at one answer position in one forward pass). The offline count through the author's own renderer and the pinned tokenizer predicted the server's numbers exactly, item by item: 744,183 input tokens over the 1,485 answered items. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.045102 per 1,000 decisions (0.698x the official cost cap USD 0.064595), against USD 0.044651 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04465098000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.045102
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "basal-omni-1.0-ultra (Remek Kinas, Gemma-4-31B-it + merged LoRA, typed-decision readout, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-17",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod, 1x NVIDIA H200 141 GB (sm_90, driver 580.173.02), Finland (EU), SPOT tier, USD 3.99/h, 17 Oct 2026; sealed run 08:59:30-09:25:01Z, one request at a time, no resumes. Speed is as measured on that card; the card's own figure is 80 ms p50 on an H200 (measured here: raw p50 0.107 s / p95 0.216 s over all 1,485 answered items; the Speed axis uses the official latency subset given in the row note, with the standing x2 + 0.15 s self-hosted adjustment on top). DEVICE DEVIATION: the pre-registered device was one non-preemptible 80 GB H100; no single-GPU H100 was rentable in the EU, so the run used this one H200 on the spot tier under a pre-registration amendment written before the sealed run (the card says about 32,000 tokens fit on a 141 GB H200 and about 10,000 on an 80 GB H100; the same 15 items were predicted to be refused on either card). Speed is therefore not strictly comparable to the H100 / RTX 5090 rows. Isolation: the author's server ran inside a network namespace with loopback only, with the cut PROVEN before any sealed item was dispatched (no interfaces but lo, no routes, outbound by name and by IP both failing) and proven again immediately before the first sealed request; credential-free environment (env -i, checked), HF offline flags, weights from a locally verified offline snapshot of the pinned revision (13 files, 62,578,671,091 bytes, every sha256 equal to the author's SHA256SUMS), sealed input held in tmpfs and deleted on the pod at 09:24:55Z, server process group killed by exact pid. Transfer: HF_HUB_DISABLE_XET=1 (plain HTTPS, same verification).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.40496740812814,
    "I_sealed": 74.71125008803229,
    "base": 74.05810874808022,
    "excess": -11.145809450259854,
    "gap": -1.3062826799041432,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 82.748926388772,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.2387543252595,
       "judge": 80.22412656558998,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 68.3768115942029,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 69.08916424140952,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 78.43444822920543,
       "hard": 75.77623771077789,
       "judge": 66.33819477685424,
       "standard": 55.168829505607754
      }
     },
     "sealed|choice": {
      "cc": 84.69020764774342,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 83.86723719208975,
       "judge": 80.60724779627816,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 72.53036721384638,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 65.45454545454545,
       "judge": 78.46153846153847,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 66.91317540250708,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 86.28980270811569,
       "hard": 61.75286075954152,
       "judge": 67.36429587432818,
       "standard": 66.86881032790222
      }
     }
    }
   },
   "jevbench_score": 71.45010731546665,
   "key": "basal-omni-1.0-ultra",
   "last_measured_on": "2026-10-17",
   "licence": "Apache-2.0 (model card and HF metadata; base google/gemma-4-31B-it Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.28",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Remek/basal-omni-1.0-ultra@b8e214cb42d0a7a4cb2c3a93b0ba02c65d7590e4",
   "noul_decisive": {
    "acc_among_decisive": 0.938953488372093,
    "decisive": 344,
    "decisive_rate": 0.9222520107238605,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.0178774866748,
     "cost": 50.37412598110784,
     "intelligence": 74.05810874808022,
     "speed": 87.31015625661291
    },
    "board_release": "v1.6.43",
    "capability": 82.53799311737751,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 3,
    "capability_rank": 6,
    "composite_ci95": {
     "A": [
      70.13672757527847,
      72.63966201572218
     ],
     "B": [
      69.95644992290508,
      73.66079468239673
     ],
     "C": [
      70.13672757527847,
      72.63966201572218
     ]
    },
    "jevbench_score": 71.69146765135073,
    "measured_in": "v1.6.28",
    "method": "v1.6",
    "open_board_rank": 6,
    "rank": 17,
    "ranks": {
     "A": 17,
     "B": 11,
     "C": 17,
     "capability": 15
    },
    "scores": {
     "A": 71.69146765135073,
     "B": 72.15261728693953,
     "C": 71.69146765135073
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3538213029736653,
     "p50_s_raw": 0.10191065148683265,
     "p95_s_adjusted": 0.5250496784574352,
     "p95_s_raw": 0.1875248392287176
    },
    "views": {
     "Balanced 33:33:33": 66.95263980712231,
     "Emphasis on Accuracy 60:20:20": 69.62468220908855,
     "Emphasis on Cost 20:20:60": 59.16409553572044,
     "Emphasis on Speed 20:60:20": 73.83928140922333,
     "Intelligence only": 74.05810874808022
    }
   },
   "rank": 19,
   "ranked": true,
   "ranks": {
    "A": 19,
    "B": 12,
    "C": 19,
    "capability": 18
   },
   "repo": "https://huggingface.co/Remek/basal-omni-1.0-ultra",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.45010731546665,
    "B": 71.95690859319264,
    "C": 71.45010731546665
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3653893249924295,
    "p50_s_raw": 0.10769466249621473,
    "p95_s_adjusted": 0.594361786666559,
    "p95_s_raw": 0.2221808933332795
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 72.80326542289306,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 18.4097962686792
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 72.80326542289306
      },
      "calibration_gen": {
       "accuracy": 0.9900990099009901,
       "cc": 94.21261550687389,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 92.0,
        "score": 90.63784652062168
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 94.21261550687389
      },
      "composite_gen": {
       "accuracy": 0.8811881188118812,
       "cc": 80.10110527775385,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 70.37037037037037,
        "score": 73.18497798321634
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 80.10110527775385
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.29139788288848,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 95.87419364866541
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.29139788288848
      },
      "jaggedness": {
       "accuracy": 0.7833333333333333,
       "cc": 68.09939011803003,
       "cc_by_type": {
        "choice": 71.12608277189607,
        "noul": 67.56756756756756,
        "score": 65.60452001462644
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 68.09939011803003
      },
      "llm_router": {
       "accuracy": 0.9265175718849841,
       "cc": 83.72918389969921,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 93.81443298969072,
        "score": 62.12789549905209
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.72918389969921
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 47.98400157140998,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 50.0,
        "score": 43.43654079670418
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 47.98400157140998
      },
      "multilingual": {
       "accuracy": 0.8072727272727273,
       "cc": 68.0930448969844,
       "cc_by_type": {
        "choice": 77.50244538637104,
        "noul": 58.33333333333333,
        "score": 68.44335597124882
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.0930448969844
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 75.63122739536792,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 53.33333333333334,
        "score": 73.5603488527704
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 75.63122739536792
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 85.46096801841463,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100.0,
        "score": 62.38891006124994
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.46096801841463
      },
      "usecase": {
       "accuracy": 0.7954545454545454,
       "cc": 69.10006746771133,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 62.5,
        "score": 66.85902593254578
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 69.10006746771133
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 38.9929915779476,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": 5.074212829080893
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 38.9929915779476
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 75.67364841521491,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 47.7582724345848
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 75.67364841521491
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.07256385800456,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 82.11149688374817
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.07256385800456
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.56994661948434,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.56272626441205
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.56994661948434
      },
      "en": {
       "accuracy": 0.8722849695916595,
       "cc": 77.45642365975384,
       "cc_by_type": {
        "choice": 87.47879571887088,
        "noul": 74.82014388489209,
        "score": 70.07033137549857
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 77.45642365975384
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 76.25205214982573,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 38.54408467133693
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 76.25205214982573
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.34203473378807,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 97.81298944726586
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.34203473378807
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.56323770395797,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 96.04265428834448
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.56323770395797
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 70.7140566079013,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 75.0,
        "score": 70.6617228963296
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.7140566079013
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 62.1449141842809,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 0.46436063546154704
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 62.1449141842809
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 57.6427804823853,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 60.00000000000001,
        "score": 27.225402606806416
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 57.6427804823853
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 42.774593132787025,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 72.76822384280551
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 42.774593132787025
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.44250882836059,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.32752648508176
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.44250882836059
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 10.262858950429482,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -35.87808981537821
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 10.262858950429482
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 65.2787917542296,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 71.0712461894768
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 65.2787917542296
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.14654979605274,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 52.867463177197706
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.14654979605274
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 51.503527144875044,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 60.04249632824212
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 51.503527144875044
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 86.36411186126297,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 80.83146601857152
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 86.36411186126297
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.41808992895669,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 94.51233430299908
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.41808992895669
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 29.716048736332187,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 22.481479542329907
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 29.716048736332187
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.884,
       "cc": 85.01693488202989,
       "cc_by_type": {
        "choice": 85.01693488202989
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 85.01693488202989
      },
      "noul": {
       "accuracy": 0.8613333333333333,
       "cc": 72.26666666666665,
       "cc_by_type": {
        "noul": 72.26666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 72.26666666666665
      },
      "score": {
       "accuracy": 0.7946666666666666,
       "cc": 68.21305822309168,
       "cc_by_type": {
        "score": 68.21305822309168
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 68.21305822309168
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 37.62962300520317,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 0.0,
        "score": 44.888869015609515
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 37.62962300520317
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 83.27418447680438,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 68.45609380308395
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 83.27418447680438
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 63.892557231272725,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100.0,
        "score": 7.9378342954441905
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 63.892557231272725
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 4.038044257628672,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -51.04376196395608
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 4.038044257628672
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 54.637068852440315,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 74.05613409355283
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 54.637068852440315
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": 22.467454630086888,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 66.66666666666667,
        "score": -61.52845371980222
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 22.467454630086888
      },
      "financial_crime": {
       "accuracy": 0.8,
       "cc": 65.815735202619,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 74.19790588796901
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 65.815735202619
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 46.54917074605785,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0.0,
        "score": 71.00639725559512
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 46.54917074605785
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 41.202351728419735,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 66.10705518525921
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 41.202351728419735
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 81.35961326233185,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100.0,
        "score": 56.92680552575358
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 81.35961326233185
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 70.4858410918331,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 45.3076524744657
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 70.4858410918331
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 14.624727606002695,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": 73.98170969972853
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 14.624727606002695
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 59.63569309338237,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 58.907079280147116
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.63569309338237
      },
      "model_routing": {
       "accuracy": 0.9032258064516129,
       "cc": 81.17230996306374,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100.0,
        "score": 50.52991690217823
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.17230996306374
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 26.178743649164545,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -100.0,
        "score": 78.53623094749362
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 26.178743649164545
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 70.2022507842471,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 77.27341901940797
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 70.2022507842471
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 61.53603210445658,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 100.0,
        "score": 28.94307168282788
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 61.53603210445658
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 47.3280686936479,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 52.12913361717557
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 47.3280686936479
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.6504026323173,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.95120789695187
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.6504026323173
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.27498047407003,
      91.59920157440426
     ],
     "I": [
      69.12512137353171,
      78.15790301804346
     ],
     "capability": [
      78.44406909592927,
      84.3637194100243
     ],
     "composite": {
      "A": [
       69.8943752475979,
       72.42598956613317
      ],
      "B": [
       69.75445190953462,
       73.46518541564177
      ],
      "C": [
       69.8943752475979,
       72.42598956613317
      ]
     },
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.20661225231586,
     "capability": 82.13236050019805,
     "intelligence": 74.05810874808022
    },
    "per_type": {
     "choice": {
      "accuracy": 0.884,
      "calibration": 90.24757479214279,
      "cc": 85.01693488202989,
      "cc_by_type": {
       "choice": 85.01693488202989
      },
      "cc_tiered": 83.71956701825772,
      "cc_tiered_open": 82.748926388772,
      "cc_tiered_sealed": 84.69020764774342,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 85.01693488202989
     },
     "noul": {
      "accuracy": 0.8613333333333333,
      "calibration": 88.94486578314562,
      "cc": 72.26666666666665,
      "cc_by_type": {
       "noul": 72.26666666666665
      },
      "cc_tiered": 70.45358940402464,
      "cc_tiered_open": 68.3768115942029,
      "cc_tiered_sealed": 72.53036721384638,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 72.26666666666665
     },
     "score": {
      "accuracy": 0.7946666666666666,
      "calibration": 91.42739618165918,
      "cc": 68.21305822309168,
      "cc_by_type": {
       "score": 68.21305822309168
      },
      "cc_tiered": 68.0011698219583,
      "cc_tiered_open": 69.08916424140952,
      "cc_tiered_sealed": 66.91317540250708,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.21305822309168
     }
    },
    "raw_on_A": {
     "calibration": 86.85140384603928,
     "capability": 80.08523448632982,
     "intelligence": 73.31906512662039
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.818926055946,
    "Emphasis on Accuracy 60:20:20": 69.53785723613201,
    "Emphasis on Cost 20:20:60": 59.10138853329149,
    "Emphasis on Speed 20:60:20": 73.54711218465104,
    "Intelligence only": 74.05810874808022
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "thegovind",
   "axes": {
    "calibration": 87.79893140350136,
    "cost": 63.5219850233163,
    "intelligence": 35.035615417279956,
    "speed": 91.21351546851116
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06420474972972842,
      "ece_score": 87.1590500540543,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.1590500540543,
      "score_before_time_limit": 87.1590500540543,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4026666666666666,
      "brier": 0.13994335801090524,
      "ece": 0.06618791977957074,
      "n": 375,
      "score": 86.76241604408585,
      "score_before_time_limit": 86.76241604408585,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.936,
      "interval90_width": 0.5479999999999998,
      "n": 375,
      "nrps": 0.11212365750633133,
      "score": 89.47532811236393,
      "score_before_time_limit": 89.95508858481628,
      "time_limit_failed_share": 0.005333333333333333,
      "top_ece": 0.04438728539867154
     }
    },
    "score": 87.79893140350136
   },
   "capability": 61.417273410390656,
   "composite_ci95": {
    "A": [
     19.884241241912253,
     37.56568680089708
    ],
    "B": [
     16.985087489256834,
     33.543106027111484
    ],
    "C": [
     13.808500862439063,
     26.087282500622976
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the author's server (output_tokens 0, nothing generated). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.016441 per 1,000 decisions (0.255x the official cost cap USD 0.064595), against USD 0.039886 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03988590000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01644121212121212
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "blink-4b (thegovind, Qwen3.5-4B derivative)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own serve.py (one forward pass, native probabilities, T 1.0).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.02146053059893,
    "I_sealed": 28.049770303960976,
    "base": 35.035615417279956,
    "excess": 4.1321634562822425,
    "gap": 13.971690226637953,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.767054755356924,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 56.59012268008808,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 23.933747412008287,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 21.739130434782616,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 40.36357942443159,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.57143972802311,
       "hard": 43.497884292247115,
       "judge": 35.671826187924616,
       "standard": 35.02866939176526
      }
     },
     "sealed|choice": {
      "cc": 58.87841680582105,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 77.12833545108006,
       "hard": 52.1221232797502,
       "judge": 60.33300685602351,
       "standard": 61.08415946002954
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 27.586206896551737,
       "hard": -1.8181818181818188,
       "judge": -23.076923076923073,
       "standard": 10.447761194029859
      }
     },
     "sealed|score": {
      "cc": 25.27089410606189,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 42.91358355022767,
       "hard": 15.290430139380662,
       "judge": 28.4348604043804,
       "standard": 31.664527869863658
      }
     }
    }
   },
   "jevbench_score": 29.472575911553054,
   "key": "blink-4b",
   "last_measured_on": "2026-10-16",
   "licence": "other (weights non-commercial, research and evaluation only)",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "thegovind/blink-4b@c243abdd2d79478db510dcdc60a88087c18547c0",
   "noul_decisive": {
    "acc_among_decisive": 0.9017857142857143,
    "decisive": 224,
    "decisive_rate": 0.5973333333333334,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.95885156098548,
     "cost": 63.5219850233163,
     "intelligence": 35.29148713463226,
     "speed": 91.30713992573452
    },
    "board_release": "v1.6.43",
    "capability": 61.62516934780887,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 57,
    "capability_rank": 82,
    "composite_ci95": {
     "A": [
      20.383405879466594,
      38.43953383528232
     ],
     "B": [
      17.414859604087905,
      34.33646076389896
     ],
     "C": [
      14.155142971851804,
      26.694120718946053
     ]
    },
    "jevbench_score": 30.012225474173363,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 69,
    "rank": 101,
    "ranks": {
     "A": 101,
     "B": 102,
     "C": 106,
     "capability": 126
    },
    "scores": {
     "A": 30.012225474173363,
     "B": 26.29431640797227,
     "C": 20.841823245953726
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.264093909971416,
     "p50_s_raw": 0.057046954985708,
     "p95_s_adjusted": 0.28023838466033335,
     "p95_s_raw": 0.06511919233016669
    },
    "views": {
     "Balanced 33:33:33": 27.15945688766263,
     "Emphasis on Accuracy 60:20:20": 22.3004177143674,
     "Emphasis on Cost 20:20:60": 28.79236405729956,
     "Emphasis on Speed 20:60:20": 32.378048784724925,
     "Intelligence only": 17.58206451384755
    }
   },
   "rank": 117,
   "ranked": true,
   "ranks": {
    "A": 117,
    "B": 118,
    "C": 126,
    "capability": 144
   },
   "repo": "https://huggingface.co/thegovind/blink-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 29.472575911553054,
    "B": 25.793034468412227,
    "C": 20.467066605245172
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2652606196701527,
    "p50_s_raw": 0.05763030983507633,
    "p95_s_adjusted": 0.28508585742674764,
    "p95_s_raw": 0.06754292871337383
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 31.2572362329405,
       "cc_by_type": {
        "choice": 41.666666666666664,
        "noul": 100.0,
        "score": -47.89495796784515
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 31.2572362329405
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 14.160560852789573,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": -12.0,
        "score": 14.927069620934985
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.160560852789573
      },
      "composite_gen": {
       "accuracy": 0.45544554455445546,
       "cc": 6.709160275028836,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -33.333333333333336,
        "score": 2.2413019632978926
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 6.709160275028836
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 82.52546910827105,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 87.57640732481315
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 82.52546910827105
      },
      "jaggedness": {
       "accuracy": 0.4444444444444444,
       "cc": 8.81363787222621,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -35.13513513513513,
        "score": 19.32388320802163
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 8.81363787222621
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 59.70781779653648,
       "cc_by_type": {
        "choice": 68.3014880690323,
        "noul": 62.88659793814433,
        "score": 47.935367382432815
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 59.70781779653648
      },
      "long_state": {
       "accuracy": 0.2,
       "cc": -103.14840833040694,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100.0,
        "score": -219.12264434605956
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -7.038566180372126,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -50.0,
        "score": 3.111105582594964
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6509090909090909,
       "cc": 34.3267274057918,
       "cc_by_type": {
        "choice": 74.5679817411151,
        "noul": -11.111111111111116,
        "score": 39.52331158737141
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.3267274057918
      },
      "negation_pair_gen": {
       "accuracy": 0.45348837209302323,
       "cc": -10.429069637362135,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -33.333333333333336,
        "score": -44.620542245419735
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.7708333333333334,
       "cc": 70.70572569906791,
       "cc_by_type": {
        "choice": 70.70572569906791
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 70.70572569906791
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 69.14502272926966,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 80.0,
        "score": 60.46810122084199
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 69.14502272926966
      },
      "usecase": {
       "accuracy": 0.5272727272727272,
       "cc": 24.832189535408105,
       "cc_by_type": {
        "choice": 57.10784313725491,
        "noul": -8.333333333333337,
        "score": 25.72205880230274
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 24.832189535408105
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 14.05020739233215,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50.0,
        "score": -12.611282584908313
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 14.05020739233215
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 32.03750075074918,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 29.445835585580895
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 32.03750075074918
      },
      "de": {
       "accuracy": 0.875,
       "cc": 78.73802460999639,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 75.0,
        "score": 69.17867559990066
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 78.73802460999639
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.81902108470916,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.16283606604554
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.81902108470916
      },
      "en": {
       "accuracy": 0.5855777584708949,
       "cc": 30.62112327914024,
       "cc_by_type": {
        "choice": 56.50529039186729,
        "noul": 8.633093525179847,
        "score": 26.72498592037359
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 30.62112327914024
      },
      "es": {
       "accuracy": 0.9354838709677419,
       "cc": 73.23800349868046,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 19.714010496041368
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.23800349868046
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -1.5898803127438204,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -33.333333333333336,
        "score": 59.71123337870843
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 39.133701489451745,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 59.754045644825815
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 39.133701489451745
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 1.584927562947776,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50.0,
        "score": -28.48544077484383
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 1.584927562947776
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 65.5019192265144,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -3.4942423204567863
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 65.5019192265144
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 33.175342862764445,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 36.71427322689542
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 33.175342862764445
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 7.694758956980356,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 23.084276870941068
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 7.694758956980356
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.56344076684583,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.3569889672042
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.56344076684583
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -5.87020332999126,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 34.337441958078166
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25.0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 30.78113910343191,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": -19.999999999999996,
        "score": 19.96085277368973
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 30.78113910343191
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 51.433523055552534,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 47.33533640295694
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 51.433523055552534
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 22.582548229673375,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -19.999999999999996,
        "score": 38.81147447625417
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 22.582548229673375
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 27.12658273445894,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0.0,
        "score": 3.118878638159428
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 27.12658273445894
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.80980266197905,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 60.42940798593717
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.80980266197905
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 5.468607851002157,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -50.260843113660194
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.468607851002157
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6893333333333334,
       "cc": 59.872940546126024,
       "cc_by_type": {
        "choice": 59.872940546126024
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.872940546126024
      },
      "noul": {
       "accuracy": 0.5386666666666666,
       "cc": 7.733333333333325,
       "cc_by_type": {
        "noul": 7.733333333333325
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 7.733333333333325
      },
      "score": {
       "accuracy": 0.49333333333333335,
       "cc": 29.087528327138855,
       "cc_by_type": {
        "score": 29.087528327138855
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 29.087528327138855
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": 0.8544195377787412,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -50.0,
        "score": 0.5632586133362261
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.8544195377787412
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 13.382698260770676,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 3.5711451618841483
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 13.382698260770676
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.70629881232147,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 49.639221640216434
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.70629881232147
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -34.52023806987325,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100.0,
        "score": -9.993462747631444
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 16.29233157470966,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 40.18134255021594
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 16.29233157470966
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -37.46002141648891,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": -100.0,
        "score": -99.80144789726546
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 25.066909912950777,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 18.898208730448964
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 25.066909912950777
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -15.544132002325737,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 26.538335700339864
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 11.219886399291537,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60.0,
        "score": 14.492992531207937
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.219886399291537
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 0.3765944167613687,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": -33.333333333333336,
        "score": -1.2970547225924056
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.3765944167613687
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 30.933446596877317,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 11.111111111111116,
        "score": 4.945042633009211
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 30.933446596877317
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -21.07845262991218,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 1.2807711425215196
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 21.659522931301932,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 31.04382791098833
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 21.659522931301932
      },
      "model_routing": {
       "accuracy": 0.7603686635944701,
       "cc": 59.538749649418754,
       "cc_by_type": {
        "choice": 70.7792207792208,
        "noul": 75.75757575757575,
        "score": 32.07945241145974
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 59.538749649418754
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -3.132727410408895,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 25.150762106585223
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 53.989306990500346,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 45.34767956017328
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 53.989306990500346
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 26.166164438071842,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0.0,
        "score": 22.83346868367365
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 26.166164438071842
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 4.004736111390193,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": 22.15913587040247
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 4.004736111390193
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.20773397171418,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 82.62320191514256
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.20773397171418
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.51478120471185,
      88.2252753118533
     ],
     "I": [
      29.913688363668427,
      38.865369340474636
     ],
     "capability": [
      56.71722361866542,
      62.745102389387604
     ],
     "composite": {
      "A": [
       19.884241241912253,
       37.56568680089708
      ],
      "B": [
       16.985087489256834,
       33.543106027111484
      ],
      "C": [
       13.808500862439063,
       26.087282500622976
      ]
     },
     "seed": 22
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.79893140350136,
     "capability": 61.417273410390656,
     "intelligence": 35.035615417279956
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6893333333333334,
      "calibration": 87.1590500540543,
      "cc": 59.872940546126024,
      "cc_by_type": {
       "choice": 59.872940546126024
      },
      "cc_tiered": 60.322735780588985,
      "cc_tiered_open": 61.767054755356924,
      "cc_tiered_sealed": 58.87841680582105,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.872940546126024
     },
     "noul": {
      "accuracy": 0.5386666666666666,
      "calibration": 86.76241604408585,
      "cc": 7.733333333333325,
      "cc_by_type": {
       "noul": 7.733333333333325
      },
      "cc_tiered": 11.966873706004144,
      "cc_tiered_open": 23.933747412008287,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 7.733333333333325
     },
     "score": {
      "accuracy": 0.49333333333333335,
      "calibration": 89.47532811236393,
      "cc": 29.087528327138855,
      "cc_by_type": {
       "score": 29.087528327138855
      },
      "cc_tiered": 32.81723676524674,
      "cc_tiered_open": 40.36357942443159,
      "cc_tiered_sealed": 25.27089410606189,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.087528327138855
     }
    },
    "raw_on_A": {
     "calibration": 84.82814231125586,
     "capability": 60.24380895524264,
     "intelligence": 35.659475599229424
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 26.661353763163415,
    "Emphasis on Accuracy 60:20:20": 21.85455941923619,
    "Emphasis on Cost 20:20:60": 28.30499284874886,
    "Emphasis on Speed 20:60:20": 31.81074393992578,
    "Intelligence only": 17.20240795670602
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AgentBull (@0xDing)",
   "axes": {
    "calibration": 84.35778065081391,
    "cost": null,
    "intelligence": 25.21389259164919,
    "speed": 90.4627802720504
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07912071749672457,
      "ece_score": 84.17585650065509,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.17585650065509,
      "score_before_time_limit": 84.17585650065509,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.7426273458445041,
      "brier": 0.16309961620902136,
      "ece": 0.09024781687300962,
      "n": 373,
      "score": 81.95043662539807,
      "score_before_time_limit": 81.95043662539807,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.7293906810035842,
      "n": 372,
      "nrps": 0.1309150904455136,
      "score": 86.9470488263886,
      "score_before_time_limit": 86.9470488263886,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0650719665133573
     }
    },
    "score": 84.35778065081391
   },
   "capability": 54.78583662123155,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 dcfcecb0...2835, and every later re-freeze, which only add hosted-API launch-list keys) RAISES on the declared base google/t5gemma-2-4b-4b ('base model lacks an explicit market-listing decision'); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's server (input 786,538, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bongard-mini (AgentBull, T5Gemma-2 4B-4B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20150, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own `bongard serve` with their temperatures.json, default token budgets (64,000 request / 32,768 sequence), bf16.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 29.129004250783858,
    "I_sealed": 21.29878093251452,
    "base": 25.21389259164919,
    "excess": -2.0093034520863746,
    "gap": 7.830223318269336,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 68.15046810038176,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 64.40342781806197,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0.0,
       "hard": -56.52173913043479,
       "judge": -50.0,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 19.23654465196982,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.916731669865264,
       "hard": 9.296542606261049,
       "judge": 38.265686721521966,
       "standard": 2.2327421301114203
      }
     },
     "sealed|choice": {
      "cc": 47.46015197187779,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 61.88055908513342,
       "hard": 40.152654099687744,
       "judge": 47.11067580803136,
       "standard": 55.38915840539971
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -31.034482758620683,
       "hard": -60.0,
       "judge": -72.3076923076923,
       "standard": -58.2089552238806
      }
     },
     "sealed|score": {
      "cc": 16.436190825665776,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 26.462361570919747,
       "hard": 11.017848733353297,
       "judge": 15.93709798260039,
       "standard": 23.008428902261812
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "bongard-mini",
   "last_measured_on": "2026-10-16",
   "licence": "Gemma terms of use",
   "listing": "listed",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "AgentBull/bongard-mini@c81c5b883a5983669be84be98658eac940489613",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base google/t5gemma-2-4b-4b and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.9375,
    "decisive": 96,
    "decisive_rate": 0.257372654155496,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.35778065081391,
     "cost": null,
     "intelligence": 25.21389259164919,
     "speed": 90.58298788933249
    },
    "board_release": "v1.6.43",
    "capability": 54.78583662123155,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2890852989861742,
     "p50_s_raw": 0.06954264949308708,
     "p95_s_adjusted": 0.3024650537088746,
     "p95_s_raw": 0.07623252685443731
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/AgentBull/bongard-mini",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28926610301132316,
    "p50_s_raw": 0.06963305150566157,
    "p95_s_adjusted": 0.31075950998929325,
    "p95_s_raw": 0.08037975499464664
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5,
       "cc": 0.4622404706696545,
       "cc_by_type": {
        "choice": 41.666666666666664,
        "noul": -50.0,
        "score": 9.7200547453423
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.4622404706696545
      },
      "calibration_gen": {
       "accuracy": 0.21782178217821782,
       "cc": -35.922342791778796,
       "cc_by_type": {
        "choice": 10.92258748674444,
        "noul": -100.0,
        "score": -18.689615862080842
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 19.530539881920774,
       "cc_by_type": {
        "choice": 80.48780487804878,
        "noul": -40.74074074074075,
        "score": 18.8445555084543
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.530539881920774
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 21.117700941050316,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -60.0,
        "score": 23.35310282315095
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 21.117700941050316
      },
      "jaggedness": {
       "accuracy": 0.26666666666666666,
       "cc": -28.736348312955844,
       "cc_by_type": {
        "choice": 16.265640038498567,
        "noul": -94.5945945945946,
        "score": -7.88009038277151
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.5942492012779552,
       "cc": 29.786361842610262,
       "cc_by_type": {
        "choice": 61.96178568283877,
        "noul": -11.340206185567014,
        "score": 38.73750603055903
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 29.786361842610262
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -20.572694854487796,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100.0,
        "score": 12.50871956024795
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5272727272727272,
       "cc": 10.133806197798606,
       "cc_by_type": {
        "choice": 60.873818063254,
        "noul": -61.111111111111114,
        "score": 30.638711641252925
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 10.133806197798606
      },
      "negation_pair_gen": {
       "accuracy": 0.6046511627906976,
       "cc": 15.634962703773246,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": -53.333333333333336,
        "score": 16.90488811131974
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 15.634962703773246
      },
      "option_order": {
       "accuracy": 0.7291666666666666,
       "cc": 65.37949400798935,
       "cc_by_type": {
        "choice": 65.37949400798935
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 65.37949400798935
      },
      "perturb_pair": {
       "accuracy": 0.71875,
       "cc": 43.09664377241094,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 19.999999999999996,
        "score": 42.32296435026585
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 43.09664377241094
      },
      "usecase": {
       "accuracy": 0.41818181818181815,
       "cc": 0.5246135780320955,
       "cc_by_type": {
        "choice": 46.07843137254903,
        "noul": -70.83333333333333,
        "score": 26.32874269488058
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.5246135780320955
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -17.34084978485645,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50.0,
        "score": -25.83207316409315
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 39.147237797438464,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 50.77504672564872
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 39.147237797438464
      },
      "de": {
       "accuracy": 0.65625,
       "cc": 43.40220333009313,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 25.0,
        "score": 45.02961883983691
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 43.40220333009313
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 19.633041242538255,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -100.0,
        "score": 75.6589002639276
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 19.633041242538255
      },
      "en": {
       "accuracy": 0.473501303214596,
       "cc": 3.7280108956536004,
       "cc_by_type": {
        "choice": 49.91518287548356,
        "noul": -53.23741007194245,
        "score": 14.506259883419691
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 3.7280108956536004
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 34.3819807991804,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 0.0,
        "score": 12.933870619400889
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 34.3819807991804
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 0.3453151180197314,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": 66.60971584586247
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.3453151180197314
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 38.936757527017214,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 37.98674316928694
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 38.936757527017214
      },
      "hi": {
       "accuracy": 0.38095238095238093,
       "cc": -13.009001358504877,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100.0,
        "score": 11.252325533423912
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 43.06232479155725,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 62.5203077080051
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 43.06232479155725
      },
      "ja": {
       "accuracy": 0.4,
       "cc": -12.03076737823514,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": -80.0,
        "score": -27.498179815404388
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -9.868365203541524,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -29.605095610624566
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.7857142857142857,
       "cc": 65.30590654320713,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 62.584386296288066
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 65.30590654320713
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -19.67037363341068,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100.0,
        "score": 18.910957021845874
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 26.281753907614387,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40.0,
        "score": 49.31500357641914
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 26.281753907614387
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 13.128874563158115,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": -19.999999999999996,
        "score": 23.028344373033573
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 13.128874563158115
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -1.2116300515052056,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 32.96085452633544
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 2.8260596941021654,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -50.0,
        "score": 23.695570386654317
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 2.8260596941021654
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 62.8586809314782,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 53.092171826692656
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.8586809314782
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -36.74882052545368,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -33.333333333333336,
        "score": -75.21821298879041
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.632,
       "cc": 52.46751755678448,
       "cc_by_type": {
        "choice": 52.46751755678448
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 52.46751755678448
      },
      "noul": {
       "accuracy": 0.24,
       "cc": -52.0,
       "cc_by_type": {
        "noul": -52.0
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.45066666666666666,
       "cc": 17.650777067912205,
       "cc_by_type": {
        "score": 17.650777067912205
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 17.650777067912205
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -14.075490238875036,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -100.0,
        "score": 21.77352928337489
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -16.706524469431017,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100.0,
        "score": 5.7810477097193775
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 20.95574183367787,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -50.0,
        "score": 61.64771330591166
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 20.95574183367787
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -32.07708838513613,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": 9.031892739328462
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": -11.862692761495312,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100.0,
        "score": 22.38293620826768
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -11.447525637999656,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -100.0,
        "score": 3.3932721426048085
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 4.739723573540634,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": 34.667349992330585
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 4.739723573540634
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -19.95542954822079,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100.0,
        "score": 23.757404735128574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 5.930574972450304,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60.0,
        "score": 61.12505825068424
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 5.930574972450304
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -0.20845375228950758,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": -66.66666666666667,
        "score": 30.281134103588304
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": 1.6257635079935728,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -77.77777777777779,
        "score": 17.53878923199106
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 1.6257635079935728
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -5.984212815810622,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 46.5634905848262
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 10.444007621596686,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -60.0,
        "score": 14.364652423331304
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 10.444007621596686
      },
      "model_routing": {
       "accuracy": 0.6082949308755761,
       "cc": 29.775167590225077,
       "cc_by_type": {
        "choice": 69.61038961038962,
        "noul": -6.060606060606055,
        "score": 25.77571922089167
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 29.775167590225077
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": -7.229669806205081,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 12.859934919196657
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 42.58687011240332,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 33.33333333333333,
        "score": 44.56655276989334
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 42.58687011240332
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -2.6708439262919,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -100.0,
        "score": 21.54411846742971
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -15.274838023083172,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -100.0,
        "score": 26.639254046692518
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.625,
       "cc": 20.18266992417227,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": -50.0,
        "score": 31.900322939776594
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 20.18266992417227
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.89481320849065,
      86.16359734422399
     ],
     "I": [
      21.17575994757959,
      28.329594719042426
     ],
     "capability": [
      51.150606303459746,
      56.002955246631714
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.35778065081391,
     "capability": 54.78583662123155,
     "intelligence": 25.21389259164919
    },
    "per_type": {
     "choice": {
      "accuracy": 0.632,
      "calibration": 84.17585650065509,
      "cc": 52.46751755678448,
      "cc_by_type": {
       "choice": 52.46751755678448
      },
      "cc_tiered": 57.805310036129775,
      "cc_tiered_open": 68.15046810038176,
      "cc_tiered_sealed": 47.46015197187779,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 52.46751755678448
     },
     "noul": {
      "accuracy": 0.24,
      "calibration": 81.95043662539807,
      "cc": -52.0,
      "cc_by_type": {
       "noul": -52.0
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.45066666666666666,
      "calibration": 86.9470488263886,
      "cc": 17.650777067912205,
      "cc_by_type": {
       "score": 17.650777067912205
      },
      "cc_tiered": 17.8363677388178,
      "cc_tiered_open": 19.23654465196982,
      "cc_tiered_sealed": 16.436190825665776,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 17.650777067912205
     }
    },
    "raw_on_A": {
     "calibration": 77.52017651507438,
     "capability": 50.95601359015508,
     "intelligence": 24.391850665235772
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Celeris",
   "axes": {
    "calibration": 88.35471243386242,
    "cost": 60.373088134904265,
    "intelligence": 58.890192862872325,
    "speed": 86.92599216009037
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05393650793650812,
      "ece_score": 89.21269841269837,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 89.21269841269837,
      "score_before_time_limit": 89.21269841269837,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.21333333333333337,
      "brier": 0.10485600000000002,
      "ece": 0.06373333333333334,
      "n": 375,
      "score": 87.25333333333334,
      "score_before_time_limit": 87.25333333333334,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9093333333333333,
      "interval90_width": 0.25799999999999995,
      "n": 375,
      "nrps": 0.07539788888888894,
      "score": 88.59810555555555,
      "score_before_time_limit": 88.59810555555555,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07632000000000008
     }
    },
    "score": 88.35471243386242
   },
   "capability": 73.62245264836737,
   "composite_ci95": {
    "A": [
     68.60424952368858,
     72.50734096171499
    ],
    "B": [
     64.96346923338149,
     70.52759161704832
    ],
    "C": [
     54.81823135353137,
     72.50734096171499
    ]
   },
   "cost": {
    "basis": "Celeris public price, USD 0.04/M input, output free (celeris.ai/pricing, fetched 16 Oct 2026 19:27Z, saved copy sha256 c3ed0fdc...; docs.celeris.ai/pricing rates table, saved copy sha256 ff4b658d..., byte-identical to the copy saved on 14 Oct 2026): \"Celeris-1 Decision $0.04 / M input tokens, Free output tokens\"; prepaid per-token billing with no subscription or commitment. Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4, Nimble and kettei), through a scoped re-freeze of the frozen pricing module that adds this one key and nothing else, with a zero-difference proof for every other registry entry. Workspace activation at Celeris is capacity-limited (an activation waitlist, as for SubQ Snap); free or promotional credit is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (1,968,382 input tokens over the 1,500 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis. Celeris counts input tokens its own way: per its docs, input_tokens \"covers everything the model reads for the request, which is more than your state and questions alone\"; on the same 1,500 items it reports about 3.0x the input tokens kettei-fat reports, so the cost reflects Celeris's own count. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.020936 per 1,000 decisions (0.324x the official cost cap USD 0.064595), against USD 0.052490 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05249018666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020936161616161617
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Celeris-1 Decision (Celeris)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted inference.celeris.ai/celeris-1-decision System One API (POST <base>/v1/systemone), model `celeris-1-decision`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `celeris` (no proxy) before the first request (custody #20414 under #19242). Native probabilities on all three question types; Celeris returns every probability at two-decimal precision (1,500 of 1,500 rows; 332 returned probabilities are exactly 0), scored as returned. The runner recorded celeris-1-decision as the answering model on all 1,500 rows (the adapter reads the response's model field). All 1,500 items answered with HTTP 200, including the 15 long items, which fit the documented 131,072-token context window (largest reported input 81,719 tokens). Paid public fast-lane order 70c4455b, run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 62.630814168468525,
    "I_sealed": 55.14957155727612,
    "base": 58.890192862872325,
    "excess": -2.358284159163304,
    "gap": 7.481242611192407,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.52874311900909,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 71.6892104435357,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 54.45410628019323,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 19.999999999999996,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 57.909593106203275,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 70.42424242424244,
       "hard": 61.16666666666667,
       "judge": 58.91525423728814,
       "standard": 43.629629629629605
      }
     },
     "sealed|choice": {
      "cc": 74.18524524536124,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 68.77529779114143,
       "judge": 72.67384916748287,
       "standard": 76.2708289390424
      }
     },
     "sealed|noul": {
      "cc": 40.147216653650005,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 58.62068965517242,
       "hard": 25.454545454545464,
       "judge": 41.53846153846155,
       "standard": 58.20895522388059
      }
     },
     "sealed|score": {
      "cc": 51.11625277281713,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.62280701754383,
       "hard": 50.50985915492958,
       "judge": 46.203125,
       "standard": 51.445454545454524
      }
     }
    }
   },
   "jevbench_score": 70.96415918238667,
   "key": "celeris-1-decision",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:inference.celeris.ai/celeris-1-decision/v1/systemone#celeris-1-decision",
   "noul_decisive": {
    "acc_among_decisive": 0.9254237288135593,
    "decisive": 295,
    "decisive_rate": 0.7866666666666666,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 13,
    "axes": {
     "calibration": 88.35471243386242,
     "cost": 60.373088134904265,
     "intelligence": 58.890192862872325,
     "speed": 86.97194897947064
    },
    "board_release": "v1.6.43",
    "capability": 73.62245264836737,
    "capability_api_board_rank": 14,
    "capability_open_board_rank": null,
    "capability_rank": 28,
    "composite_ci95": {
     "A": [
      68.61140285654639,
      72.51533144283196
     ],
     "B": [
      64.96860049103343,
      70.53363954277505
     ],
     "C": [
      54.823943816415806,
      72.51533144283196
     ]
    },
    "jevbench_score": 70.97181314040431,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 20,
    "ranks": {
     "A": 20,
     "B": 28,
     "C": 30,
     "capability": 52
    },
    "scores": {
     "A": 70.97181314040431,
     "B": 68.174542834176,
     "C": 68.3705939780874
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4326209495193325,
     "p50_s_raw": 0.4326209495193325,
     "p95_s_adjusted": 0.46419189785665366,
     "p95_s_raw": 0.46419189785665366
    },
    "views": {
     "Balanced 33:33:33": 66.60392918110978,
     "Emphasis on Accuracy 60:20:20": 63.28801502769513,
     "Emphasis on Cost 20:20:60": 63.96337756642816,
     "Emphasis on Speed 20:60:20": 73.48801217314741,
     "Intelligence only": 58.89019286287232
    }
   },
   "rank": 22,
   "ranked": true,
   "ranks": {
    "A": 22,
    "B": 30,
    "C": 33,
    "capability": 63
   },
   "repo": "https://docs.celeris.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.96415918238667,
    "B": 68.16889268941678,
    "C": 68.36322054865403
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.43277494149515405,
    "p50_s_raw": 0.43277494149515405,
    "p95_s_adjusted": 0.46896310617739795,
    "p95_s_raw": 0.46896310617739795
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 88.55555555555556,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 74.0
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 88.55555555555556
      },
      "calibration_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 56.85813229056203,
       "cc_by_type": {
        "choice": 77.7306468716861,
        "noul": 36.00000000000001,
        "score": 56.84374999999999
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 56.85813229056203
      },
      "composite_gen": {
       "accuracy": 0.8613861386138614,
       "cc": 69.98257012484656,
       "cc_by_type": {
        "choice": 86.9918699186992,
        "noul": 48.14814814814814,
        "score": 74.80769230769232
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 69.98257012484656
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.5925925925926,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 96.77777777777777
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.5925925925926
      },
      "jaggedness": {
       "accuracy": 0.6222222222222222,
       "cc": 36.09978545606696,
       "cc_by_type": {
        "choice": 59.57651588065449,
        "noul": 18.918918918918926,
        "score": 29.803921568627445
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 36.09978545606696
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 68.70445180122648,
       "cc_by_type": {
        "choice": 74.64119045522585,
        "noul": 69.0721649484536,
        "score": 62.40000000000001
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.70445180122648
      },
      "long_state": {
       "accuracy": 0.4666666666666667,
       "cc": -28.00537634408603,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -19.50000000000003
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 14.397316314842081,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -25.0,
        "score": 30.047619047619033
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 14.397316314842081
      },
      "multilingual": {
       "accuracy": 0.7163636363636363,
       "cc": 56.70515248138471,
       "cc_by_type": {
        "choice": 70.6553635474405,
        "noul": 41.66666666666667,
        "score": 57.793427230046944
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 56.70515248138471
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 50.21111111111111,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 19.999999999999996,
        "score": 37.3
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 50.21111111111111
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 82.92889042889043,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100.0,
        "score": 69.8076923076923
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.92889042889043
      },
      "usecase": {
       "accuracy": 0.7045454545454546,
       "cc": 52.40729411764707,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 41.66666666666667,
        "score": 42.51600000000001
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.40729411764707
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 10.50061050061049,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50.0,
        "score": -42.307692307692335
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 10.50061050061049
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 46.74444444444443,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 6.899999999999984
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 46.74444444444443
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.51365909965371,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 86.43478260869566
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.51365909965371
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 54.66014897579143,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 97.49999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 54.66014897579143
      },
      "en": {
       "accuracy": 0.7654213727193745,
       "cc": 59.11116852992231,
       "cc_by_type": {
        "choice": 76.71495344211078,
        "noul": 46.76258992805755,
        "score": 53.85596221959859
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 59.11116852992231
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 59.20020471483863,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 75.0,
        "score": 22.176470588235276
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 59.20020471483863
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 22.102003642987246,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 97.99999999999999
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 22.102003642987246
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.71310629514964,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 75.31578947368422
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.71310629514964
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 51.663407821229065,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25.0,
        "score": 46.750000000000014
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 51.663407821229065
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 71.551683183016,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 42.71428571428573
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.551683183016
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 43.147383284793925,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 32.33333333333333
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 43.147383284793925
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 23.642857142857128,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 70.9285714285714
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 23.642857142857128
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.16666666666667,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 98.16666666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.16666666666667
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 33.964213564213566,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 61.19999999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.964213564213566
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 56.32830097071453,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 39.99999999999999,
        "score": 51.83720930232556
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.32830097071453
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 50.6765829642443,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 45.06451612903225
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 50.6765829642443
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 63.15602836879433,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 55.00000000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 63.15602836879433
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 65.375,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 46.12500000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 65.375
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.99462365591397,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 92.49999999999999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.99462365591397
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 46.111111111111114,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 5.000000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.111111111111114
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8093333333333333,
       "cc": 75.37266308195717,
       "cc_by_type": {
        "choice": 75.37266308195717
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.37266308195717
      },
      "noul": {
       "accuracy": 0.728,
       "cc": 45.599999999999994,
       "cc_by_type": {
        "noul": 45.599999999999994
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 45.599999999999994
      },
      "score": {
       "accuracy": 0.6693333333333333,
       "cc": 53.62184115523472,
       "cc_by_type": {
        "score": 53.62184115523472
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 53.62184115523472
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": -3.3333333333333335,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -100.0,
        "score": 5.999999999999995
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 60.539178213091255,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100.0,
        "score": 18.884615384615376
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 60.539178213091255
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 53.159891598916,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 41.99999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 53.159891598916
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -5.883528265107214,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 9.250000000000004
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 38.08121980676328,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 76.56249999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 38.08121980676328
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 25.78092243186579,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 100.0,
        "score": -97.5000000000001
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 25.78092243186579
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 34.35667600373482,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 27.999999999999993
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.35667600373482
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 25.161793667626117,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50.0,
        "score": 46.39130434782608
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 25.161793667626117
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": -0.9074074074074095,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 19.77777777777777
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 35.525944671554946,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 24.636363636363644
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 35.525944671554946
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 53.53648892020985,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 16.681818181818187
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 53.53648892020985
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 61.327956989247305,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 48.499999999999986
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 61.327956989247305
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 64.07245681381958,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 55.24999999999998
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 64.07245681381958
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 75.8801857864358,
       "cc_by_type": {
        "choice": 89.48051948051949,
        "noul": 87.87878787878789,
        "score": 50.28125000000002
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.8801857864358
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 19.180836250047037,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 69.05882352941175
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 19.180836250047037
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 58.28367021403233,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 58.23076923076924
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 58.28367021403233
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 27.49618468076886,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0.0,
        "score": 26.823529411764714
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 27.49618468076886
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 54.415942028985505,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 58.9
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 54.415942028985505
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.34210526315788,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 83.02631578947367
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.34210526315788
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.97879030820103,
      89.7941902777778
     ],
     "I": [
      53.60515070087634,
      63.490315354109036
     ],
     "capability": [
      69.55833052839986,
      75.84428813904887
     ],
     "composite": {
      "A": [
       68.60424952368858,
       72.50734096171499
      ],
      "B": [
       64.96346923338149,
       70.52759161704832
      ],
      "C": [
       54.81823135353137,
       72.50734096171499
      ]
     },
     "seed": 24
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.35471243386242,
     "capability": 73.62245264836737,
     "intelligence": 58.890192862872325
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 89.21269841269837,
      "cc": 75.37266308195717,
      "cc_by_type": {
       "choice": 75.37266308195717
      },
      "cc_tiered": 74.85699418218516,
      "cc_tiered_open": 75.52874311900909,
      "cc_tiered_sealed": 74.18524524536124,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.37266308195717
     },
     "noul": {
      "accuracy": 0.728,
      "calibration": 87.25333333333334,
      "cc": 45.599999999999994,
      "cc_by_type": {
       "noul": 45.599999999999994
      },
      "cc_tiered": 47.30066146692162,
      "cc_tiered_open": 54.45410628019323,
      "cc_tiered_sealed": 40.147216653650005,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 45.599999999999994
     },
     "score": {
      "accuracy": 0.6693333333333333,
      "calibration": 88.59810555555555,
      "cc": 53.62184115523472,
      "cc_by_type": {
       "score": 53.62184115523472
      },
      "cc_tiered": 54.512922939510204,
      "cc_tiered_open": 57.909593106203275,
      "cc_tiered_sealed": 51.11625277281713,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.62184115523472
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.59494163888954,
    "Emphasis on Accuracy 60:20:20": 63.28314579381391,
    "Emphasis on Cost 20:20:60": 63.95840386047157,
    "Emphasis on Speed 20:60:20": 73.46832021219848,
    "Intelligence only": 58.89019286287232
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Ninh (ninhnv / xoai)",
   "axes": {
    "calibration": 58.650234835105636,
    "cost": 48.98575014908867,
    "intelligence": 27.33769023776734,
    "speed": 64.6591918236347
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04707418051854133,
      "ece_score": 90.58516389629173,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 56.76670270834282,
      "score_before_time_limit": 90.58516389629173,
      "time_limit_failed_share": 0.37333333333333335
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.08477561036707809,
      "ece": 0.03417779219939911,
      "n": 375,
      "score": 68.81746749907545,
      "score_before_time_limit": 93.16444156012018,
      "time_limit_failed_share": 0.2613333333333333
     },
     "score": {
      "interval90_coverage": 0.9146666666666666,
      "interval90_width": 0.18444444444444447,
      "n": 375,
      "nrps": 0.05480603059927663,
      "score": 50.36653429789865,
      "score_before_time_limit": 88.67347587658213,
      "time_limit_failed_share": 0.432,
      "top_ece": 0.08586222593454032
     }
    },
    "score": 58.650234835105636
   },
   "capability": 42.993962536436484,
   "composite_ci95": {
    "A": [
     8.4328219014699,
     18.339078734177004
    ],
    "B": [
     7.272301635276294,
     16.692902424806153
    ],
    "C": [
     5.8561263204652105,
     12.735471343178475
    ]
   },
   "cost": {
    "basis": "ESTIMATE: base-model reference price per v1.5-M2 section 2 with INTERPRETATION-1 I-3 (frozen 25 Sep 2026 cut-off snapshots). The exact base google/gemma-4-12B-it (adapter_config base_model_name_or_path, chon_config base_model; pinned commit 707f0a3b) has no listing on OpenRouter or DeepInfra in those snapshots (OPENROUTER-MODELS-20260925.json SHA-256 f5d0cd8e0f7d2d6514748c99bfcaeacc09775bac7aa7eecf765af4ba04b0dcd5; DEEPINFRA-MODELS-20260925.json SHA-256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f) nor in the order's OpenRouter snapshot read 2026-10-19T03:50:11Z, so the nearest listed sibling of the same family and size, OpenRouter google/gemma-3-12b-it, is used: USD 0.05/M input, USD 0.15/M output (DeepInfra lists the same). Same basis as the Cygnet, Winnow-12B, Jev-Omni, Torchcast-Decision-12B and Metask-12B rows. Applied to the server's own usage: input tokens encoded and output tokens generated by its think-when-unsure pass (up to 512 per decision). Self-hosted, no bookable tariff; estimated, not charged. COMMON COST BASIS: published cost is recomputed over the same remaining 1,485 items as every other row; the 15 long states are answered and billed too. All 1,500 items count for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05329780000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05017356902356902
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 540,
    "n_over_limit_answered_ok": 540,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Chon-1.0-12B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-19",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Verda GPU VM (RTX 6000 Ada), offline read-only container (--network none); author's chon.serve unchanged (xoai/chon @09ab5fa4) with the adapter's chon_config.json (bf16 LoRA r64 on the pinned google/gemma-4-12B-it @707f0a3b, letter readout, per-type calibration, yes/no commit rule, think-when-unsure pass up to 512 tokens by the base model for at most 45 % of questions) on the documented stack torch 2.9.1+cu128 / transformers 5.20.1 / peft 0.18.1",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.05725156238466,
    "I_sealed": 22.618128913150024,
    "base": 27.33769023776734,
    "excess": -0.4004041211210776,
    "gap": 9.439122649234633,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 50.8963923683134,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 45.265806857502355,
       "judge": 44.62755438365195,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 45.27536231884058,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -0.7618837174541815,
       "hard": -34.17965289489457,
       "judge": -54.65475090077187,
       "standard": -32.428756510153846
      }
     },
     "sealed|choice": {
      "cc": 42.8028155622999,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 63.78653113087674,
       "hard": 34.94853706487798,
       "judge": 43.58472086190012,
       "standard": 46.84665682345498
      }
     },
     "sealed|noul": {
      "cc": 25.05157117715018,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 16.36363636363636,
       "judge": 10.769230769230775,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -18.093529139951947,
       "hard": -63.26646179684954,
       "judge": -58.618791804360825,
       "standard": -38.47436169952119
      }
     }
    }
   },
   "jevbench_score": 12.822528228783156,
   "key": "chon-1.0-12b",
   "last_measured_on": "2026-10-19",
   "licence": "Apache-2.0 (adapter); base google/gemma-4-12B-it Apache-2.0; serving package xoai/chon Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.32",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ninhnv/chon-1.0-12b@cf3c0d3d5aa3230cbd36959cc4ed257b41000c5e",
   "noul_decisive": {
    "acc_among_decisive": 0.9025270758122743,
    "decisive": 277,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 277
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.80769377766468,
     "cost": 48.98575014908867,
     "intelligence": 79.86402006880415,
     "speed": 65.53984816070655
    },
    "board_release": "v1.6.43",
    "capability": 85.33585692323442,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 1,
    "capability_rank": 2,
    "composite_ci95": {
     "A": [
      63.87535788763423,
      65.45251851082296
     ],
     "B": [
      65.55109890484802,
      67.8391153066576
     ],
     "C": [
      63.87535788763423,
      65.45251851082296
     ]
    },
    "jevbench_score": 64.84806528194059,
    "measured_in": "v1.6.32",
    "method": "v1.6",
    "open_board_rank": 28,
    "rank": 49,
    "ranks": {
     "A": 49,
     "B": 34,
     "C": 36,
     "capability": 7
    },
    "scores": {
     "A": 64.84806528194059,
     "B": 66.90950425070145,
     "C": 64.84806528194059
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4174180539997906,
     "p50_s_raw": 0.1337090269998953,
     "p95_s_adjusted": 66.9027477348007,
     "p95_s_raw": 33.376373867400346
    },
    "views": {
     "Balanced 33:33:33": 59.74950639807396,
     "Emphasis on Accuracy 60:20:20": 65.5308560631613,
     "Emphasis on Cost 20:20:60": 53.91067018855768,
     "Emphasis on Speed 20:60:20": 60.97401349973243,
     "Intelligence only": 76.65679978603575
    }
   },
   "rank": 183,
   "ranked": true,
   "ranks": {
    "A": 183,
    "B": 185,
    "C": 185,
    "capability": 219
   },
   "repo": "https://huggingface.co/ninhnv/chon-1.0-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 12.822528228783156,
    "B": 11.378252987205936,
    "C": 8.904533492210525
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5102759639998112,
    "p50_s_raw": 0.1801379819999056,
    "p95_s_adjusted": 67.0310011336996,
    "p95_s_raw": 33.4405005668498
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": -36.7117256790645,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": -285.1351770371935
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.45544554455445546,
       "cc": -6.534473748024702,
       "cc_by_type": {
        "choice": 17.285259809119836,
        "noul": 43.99999999999999,
        "score": -80.88868105319393
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.4752475247524752,
       "cc": -39.39465509734739,
       "cc_by_type": {
        "choice": 60.975609756097555,
        "noul": -55.55555555555556,
        "score": -123.60401949258417
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 77.75646179758371,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 33.26938539275112
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.75646179758371
      },
      "jaggedness": {
       "accuracy": 0.3055555555555556,
       "cc": -40.08141520361054,
       "cc_by_type": {
        "choice": 6.15976900866219,
        "noul": -8.108108108108103,
        "score": -118.29590651138572
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6996805111821086,
       "cc": 43.778009451448945,
       "cc_by_type": {
        "choice": 55.622083296645236,
        "noul": 71.13402061855669,
        "score": 4.577924439144898
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 43.778009451448945
      },
      "long_state": {
       "accuracy": 0.06666666666666667,
       "cc": -104.16096967236842,
       "cc_by_type": {
        "choice": -16.129032258064516,
        "noul": -100.0,
        "score": -196.3538767590408
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -30.212519107355362,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": 0.0,
        "score": -91.66848515711763
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.64,
       "cc": 30.49680134970566,
       "cc_by_type": {
        "choice": 53.04858167590479,
        "noul": 47.22222222222223,
        "score": -8.780399849010045
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.49680134970566
      },
      "negation_pair_gen": {
       "accuracy": 0.4069767441860465,
       "cc": -70.10562596759534,
       "cc_by_type": {
        "choice": 43.33333333333332,
        "noul": -46.666666666666664,
        "score": -206.9835445694527
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.625,
       "cc": 52.06391478029295,
       "cc_by_type": {
        "choice": 52.06391478029295
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 52.06391478029295
      },
      "perturb_pair": {
       "accuracy": 0.734375,
       "cc": 64.30687421060522,
       "cc_by_type": {
        "choice": 60.96096096096097,
        "noul": 100.0,
        "score": 31.9596616708547
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 64.30687421060522
      },
      "usecase": {
       "accuracy": 0.55,
       "cc": 15.990260307696246,
       "cc_by_type": {
        "choice": 48.529411764705905,
        "noul": 33.33333333333333,
        "score": -33.89196417495049
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 15.990260307696246
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -44.477995629455364,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50.0,
        "score": -188.19589165027085
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 0.04504707569335409,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": -91.72353772837307
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.04504707569335409
      },
      "de": {
       "accuracy": 0.75,
       "cc": 50.00742718629285,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100.0,
        "score": -10.154709591563925
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 50.00742718629285
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 44.241783207348895,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": 0.0,
        "score": 99.76445576729809
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.241783207348895
      },
      "en": {
       "accuracy": 0.5438748913987836,
       "cc": 5.935837840547715,
       "cc_by_type": {
        "choice": 43.3250753590998,
        "noul": 26.618705035971214,
        "score": -52.13626687342787
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 5.935837840547715
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 33.860230145759715,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75.0,
        "score": -63.63138134086115
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 33.860230145759715
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 44.86609847866006,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 99.62561784035175
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 44.86609847866006
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 62.13995588663956,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100.0,
        "score": 49.94927942462455
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 62.13995588663956
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 28.235116753398046,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 50.0,
        "score": -15.015320130867305
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 28.235116753398046
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.6666666666666666,
       "cc": 24.05542687542328,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -99.77448320849243
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 24.05542687542328
      },
      "ja": {
       "accuracy": 0.6,
       "cc": -19.39457564965358,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 60.00000000000001,
        "score": -175.29254347000918
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -15.954208575210638,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -47.862625725631915
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.67239947898015,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.68386510360713
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.67239947898015
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -21.953877958184364,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": -80.58024859316781
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 35.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 35.483870967741936
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 62.28121798567983,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 80.0,
        "score": 29.695960347221483
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 62.28121798567983
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 28.62182908520281,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": -61.09974550809223
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 28.62182908520281
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": -18.865248226950353,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": -100.0
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": -21.306872210115102,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": 50.0,
        "score": -126.96409489121487
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 34.37659706332849,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 100.0,
        "score": -0.09601526162743568
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 34.37659706332849
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -28.461345098112986,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -101.20324433388697
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5773333333333334,
       "cc": 45.40653284601696,
       "cc_by_type": {
        "choice": 45.40653284601696
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 45.40653284601696
      },
      "noul": {
       "accuracy": 0.6666666666666666,
       "cc": 33.33333333333333,
       "cc_by_type": {
        "noul": 33.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 33.33333333333333
      },
      "score": {
       "accuracy": 0.43466666666666665,
       "cc": -46.983912693513005,
       "cc_by_type": {
        "score": -46.983912693513005
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -33.51713526605247,
       "cc_by_type": {
        "choice": 4.0,
        "noul": -75.0,
        "score": -29.551405798157404
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 23.602515742084368,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 100.0,
        "score": -73.29183165573447
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 23.602515742084368
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": -17.11044795470467,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 50.0,
        "score": -120.03053085598393
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -91.76413255360625,
       "cc_by_type": {
        "choice": -16.959064327485383,
        "noul": -33.333333333333336,
        "score": -225.0
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": -4.135618112106002,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": -35.59526013341945
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -132.30072112549024,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 66.66666666666667,
        "score": -500.6757482821311
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 10.927112300993505,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": 14.28571428571428,
        "score": -32.48476953959651
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 10.927112300993505
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -59.36527554403482,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": -104.92509492478739
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 30.276690027327678,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 19.999999999999996,
        "score": 74.99673674864971
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 30.276690027327678
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": -11.981853759739328,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 33.33333333333333,
        "score": -79.3431344412451
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 38.2483397949752,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": -29.182629193885766
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 38.2483397949752
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -34.214963590681506,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -139.20403055699074
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 59.95162000305285,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 19.854860009158525
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.95162000305285
      },
      "model_routing": {
       "accuracy": 0.7741935483870968,
       "cc": 47.412797926009404,
       "cc_by_type": {
        "choice": 75.45454545454547,
        "noul": 93.93939393939394,
        "score": -27.1555456159112
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 47.412797926009404
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -37.366321374133996,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": -54.517390226048825
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 34.71062516102369,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": -79.15503259492336
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 34.71062516102369
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -30.2229358544971,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": -81.55550707088047
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -13.89812014922775,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 0.0,
        "score": -54.73783870855282
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 88.31407046530585,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 75.61836797954744
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 88.31407046530585
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.21584329874216,
      60.88788711420117
     ],
     "I": [
      23.051062243612662,
      31.745239730528457
     ],
     "capability": [
      39.744609791998755,
      45.82160099193017
     ],
     "composite": {
      "A": [
       8.4328219014699,
       18.339078734177004
      ],
      "B": [
       7.272301635276294,
       16.692902424806153
      ],
      "C": [
       5.8561263204652105,
       12.735471343178475
      ]
     },
     "seed": 25
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 58.650234835105636,
     "capability": 42.993962536436484,
     "intelligence": 27.33769023776734
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5773333333333334,
      "calibration": 56.76670270834282,
      "cc": 45.40653284601696,
      "cc_by_type": {
       "choice": 45.40653284601696
      },
      "cc_tiered": 46.84960396530665,
      "cc_tiered_open": 50.8963923683134,
      "cc_tiered_sealed": 42.8028155622999,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 45.40653284601696
     },
     "noul": {
      "accuracy": 0.6666666666666666,
      "calibration": 68.81746749907545,
      "cc": 33.33333333333333,
      "cc_by_type": {
       "noul": 33.33333333333333
      },
      "cc_tiered": 35.16346674799538,
      "cc_tiered_open": 45.27536231884058,
      "cc_tiered_sealed": 25.05157117715018,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.33333333333333
     },
     "score": {
      "accuracy": 0.43466666666666665,
      "calibration": 50.36653429789865,
      "cc": -46.983912693513005,
      "cc_by_type": {
       "score": -46.983912693513005
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 57.974564399987855,
     "capability": 42.74401838693816,
     "intelligence": 27.51347237388846
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.3733333333333333,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.2613333333333333,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.43200000000000005,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 11.879824484728795,
    "Emphasis on Accuracy 60:20:20": 9.852282663726074,
    "Emphasis on Cost 20:20:60": 12.664002933738253,
    "Emphasis on Speed 20:60:20": 13.876229078418996,
    "Intelligence only": 7.84413326697842
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Ninh (ninhnv / xoai)",
   "axes": {
    "calibration": 81.82393852673533,
    "cost": 70.74636118712607,
    "intelligence": 54.96148352069754,
    "speed": 43.95627533510597
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09101480224243556,
      "ece_score": 81.79703955151288,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 81.79703955151288
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.15139765553945303,
      "ece": 0.09557442624031161,
      "n": 375,
      "score": 80.88511475193768
     },
     "score": {
      "interval90_coverage": 0.8906666666666667,
      "interval90_width": 0.3764444444444446,
      "n": 375,
      "nrps": 0.10648183537819128,
      "score": 82.78966127675545,
      "top_ece": 0.1188624695433499
     }
    },
    "score": 81.82393852673533
   },
   "capability": 68.39271102371643,
   "composite_ci95": {
    "A": [
     44.392703292458805,
     46.89203249517508
    ],
    "B": [
     43.17507195851998,
     46.534331841823644
    ],
    "C": [
     31.09870948973084,
     44.91059204066695
    ]
   },
   "cost": {
    "basis": "Exact listed rate of the base model google/gemma-4-E4B-it (DeepInfra, frozen 25 Sep 2026 catalog snapshot, USD 0.02/M input, 0.10/M output); this line is added to the frozen pricing module at its next re-freeze. No output tokens under the author's --no-think mode. COMMON COST BASIS: the published cost uses the same remaining 1,485 items as every other row: 701,161 input tokens, USD 0.009443245791245792 per 1,000 decisions (0.146x cap). ALL-ITEMS DIAGNOSTIC: USD 0.010571213333333333 per 1,000 over all 1,500 items (0.164x cap). All items still count for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.010571213333333334
    },
    "kind": "listed",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009443245791245792
   },
   "decision_limit": null,
   "display": "Chon-1.0-E4B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-19",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Our own CPU server (AMD EPYC, 32 cores), disposable container with no network; the network cut was proven inside the container before it read any sealed item. Served by the author's own chon.serve 1.0.0 in its documented CPU mode (--device cpu --no-think). Published Speed applies the standard self-hosted x2 + 0.15 s adjustment (assumption, not measured).",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 59.11785286864142,
    "I_sealed": 50.805114172753655,
    "base": 54.96148352069754,
    "excess": -1.5267880744679445,
    "gap": 8.312738695887766,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 62.113835115843074,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 64.40342781806197,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 71.3768115942029,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 43.862911895878305,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.71354470950146,
       "hard": 37.50543489517916,
       "judge": 57.48103056832748,
       "standard": 30.725371481791253
      }
     },
     "sealed|choice": {
      "cc": 63.02647928723724,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 55.76500520411702,
       "judge": 62.97747306562194,
       "standard": 71.52499472685088
      }
     },
     "sealed|noul": {
      "cc": 55.309373076743114,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 54.54545454545454,
       "judge": 50.76923076923077,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 34.079490154280634,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 44.05642671414762,
       "hard": 32.35747991578803,
       "judge": 33.25594585627047,
       "standard": 33.77035879834759
      }
     }
    }
   },
   "jevbench_score": 45.93476445894657,
   "key": "chon-1.0-e4b",
   "last_measured_on": "2026-10-19",
   "licence": "Apache-2.0 (adapter card and base licence agree)",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.32",
   "measurement_date_status": "measured",
   "method": "v1.6",
   "model_pin": "ninhnv/chon-1.0-e4b@b7b9ecdaa25d83696fb1c82c3b0acaff2923137a",
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": 0.7946666666666666,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.82393852673533,
     "cost": 70.74636118712607,
     "intelligence": 54.96148352069754,
     "speed": 43.95627533510597
    },
    "board_release": "v1.6.43",
    "capability": 68.39271102371643,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      44.392703292458805,
      46.89203249517508
     ],
     "B": [
      43.17507195851998,
      46.534331841823644
     ],
     "C": [
      31.09870948973084,
      44.91059204066695
     ]
    },
    "jevbench_score": 45.93476445894657,
    "measured_in": "v1.6.32",
    "method": "v1.6",
    "open_board_rank": 44,
    "rank": 69,
    "ranks": {
     "A": 69,
     "B": 67,
     "C": 67,
     "capability": 87
    },
    "scores": {
     "A": 45.93476445894657,
     "B": 45.19903845098299,
     "C": 38.5439204552548
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 44.504827468981965,
     "p50_s_raw": 22.177413734490983,
     "p95_s_adjusted": 90.35773683558801,
     "p95_s_raw": 45.103868417794004
    },
    "views": {
     "Balanced 33:33:33": 42.095287401061526,
     "Emphasis on Accuracy 60:20:20": 42.2473928103636,
     "Emphasis on Cost 20:20:60": 46.3627340067867,
     "Emphasis on Speed 20:60:20": 38.42054978162839,
     "Intelligence only": 42.477623199386635
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/ninhnv/chon-1.0-e4b",
   "scores": {
    "A": 45.93476445894657,
    "B": 45.19903845098299,
    "C": 38.5439204552548
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 44.504827468981965,
    "p50_s_raw": 22.177413734490983,
    "p95_s_adjusted": 90.35773683558801,
    "p95_s_raw": 45.103868417794004
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 69.10800085114204,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 23.99066922009281
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 69.10800085114204
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 27.39797571103509,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 43.99999999999999,
        "score": 1.8206503568592547
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 27.39797571103509
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 33.0387450611331,
       "cc_by_type": {
        "choice": 64.22764227642277,
        "noul": 3.703703703703698,
        "score": 31.184889203272835
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 33.0387450611331
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 75.50614497103835,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 66.51843491311506
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 75.50614497103835
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 28.31724015648845,
       "cc_by_type": {
        "choice": 49.4706448508181,
        "noul": 35.13513513513513,
        "score": 0.34594048351211937
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 28.31724015648845
      },
      "llm_router": {
       "accuracy": 0.7987220447284346,
       "cc": 68.96682037997233,
       "cc_by_type": {
        "choice": 65.92409967420974,
        "noul": 87.62886597938144,
        "score": 53.34749548632583
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.96682037997233
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 9.378307964358866,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0,
        "score": 18.45750453823789
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 9.378307964358866
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 22.828236669030773,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": 25,
        "score": 17.711514130803664
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 22.828236669030773
      },
      "multilingual": {
       "accuracy": 0.7636363636363637,
       "cc": 64.05836316914788,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": 66.66666666666667,
        "score": 55.83121384175516
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 64.05836316914788
      },
      "negation_pair_gen": {
       "accuracy": 0.5813953488372093,
       "cc": 24.426752663997473,
       "cc_by_type": {
        "choice": 50,
        "noul": 13.33333333333333,
        "score": 9.946924658659084
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 24.426752663997473
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 69.51839721905846,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 38.5852216872054
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 69.51839721905846
      },
      "usecase": {
       "accuracy": 0.7227272727272728,
       "cc": 62.18107390251658,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 70.83333333333333,
        "score": 42.6706726879419
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 62.18107390251658
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 25.547602788904285,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 100,
        "score": -28.1190963951919
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 25.547602788904285
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 28.902401515803394,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 40.777865069683614
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 28.902401515803394
      },
      "de": {
       "accuracy": 0.875,
       "cc": 82.54988241544932,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 79.50805432599395
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 82.54988241544932
      },
      "el": {
       "accuracy": 0.6,
       "cc": 40.72667100867935,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 89.03289943199708
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 40.72667100867935
      },
      "en": {
       "accuracy": 0.6750651607298002,
       "cc": 49.948331118583404,
       "cc_by_type": {
        "choice": 62.43638715661266,
        "noul": 55.395683453237424,
        "score": 32.01292274590012
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 49.948331118583404
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 68.23881788573766,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 4.716453657212982
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.23881788573766
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 75.46287033873422,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100,
        "score": 91.9623815080059
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 75.46287033873422
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 73.32904837911452,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 83.51655690204947
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 73.32904837911452
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 61.5339984543601,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 50,
        "score": 51.36177189939314
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 61.5339984543601
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 71.82276559964139,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 29.497914881542997
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.82276559964139
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 44.97147892919882,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 80,
        "score": 26.39974258584905
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 44.97147892919882
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 76.88577914875698,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 30.657337446270972
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 76.88577914875698
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.2462144174098,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.73864325222938
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.2462144174098
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 45.84212257699509,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 30.16706037167789
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 45.84212257699509
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 68.0579119218342,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 51.79117122889663
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 68.0579119218342
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 75.20820126987579,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 68.05241759866685
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.20820126987579
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 41.70136243631472,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 41.699831989795186
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 41.70136243631472
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 40.435388558025224,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": 14.784426543640894
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 40.435388558025224
      },
      "uk": {
       "accuracy": 1,
       "cc": 94.04904312421029,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 82.14712937263086
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 94.04904312421029
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 23.097381146585246,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": -13.193732266458946
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 23.097381146585246
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7186666666666667,
       "cc": 63.661761610440315,
       "cc_by_type": {
        "choice": 63.661761610440315
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 63.661761610440315
      },
      "noul": {
       "accuracy": 0.7946666666666666,
       "cc": 58.93333333333332,
       "cc_by_type": {
        "noul": 58.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 58.93333333333332
      },
      "score": {
       "accuracy": 0.5466666666666666,
       "cc": 36.46304903717563,
       "cc_by_type": {
        "score": 36.46304903717563
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.46304903717563
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 60.11455074722698,
       "cc_by_type": {
        "choice": 68,
        "noul": 50,
        "score": 62.343652241680935
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 60.11455074722698
      },
      "code_linting": {
       "accuracy": 0.7083333333333334,
       "cc": 52.98927870838881,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 77.77777777777777,
        "score": -0.17640127994054122
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 52.98927870838881
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 63.07909063485587,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 88.01775970944567
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 63.07909063485587
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 1.3405683540335727,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -33.333333333333336,
        "score": 7.530476991925283
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 1.3405683540335727
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 22.50908469774433,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 29.846094672943135
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.50908469774433
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -2.182225061345368,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100,
        "score": -156.23220977523107
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.8,
       "cc": 65.99713161742129,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 71.42857142857142,
        "score": 36.36674499231988
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 65.99713161742129
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 41.51345502838387,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": 50,
        "score": 37.258135119994826
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 41.51345502838387
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 55.0828385149075,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 46.91518221138916
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 55.0828385149075
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 39.60594005800316,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 16.390982201132953
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 39.60594005800316
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 59.68092147823734,
       "cc_by_type": {
        "choice": 100,
        "noul": 55.55555555555556,
        "score": 23.48720887915645
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 59.68092147823734
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 26.22694115574735,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 9.863619166166792
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 26.22694115574735
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.69185757191399,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 57.59188749501257
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.69185757191399
      },
      "model_routing": {
       "accuracy": 0.8202764976958525,
       "cc": 69.08316612831908,
       "cc_by_type": {
        "choice": 74.28571428571429,
        "noul": 93.93939393939394,
        "score": 39.024390159849
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.08316612831908
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 77.16862866609263,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 100,
        "score": 66.0548303360898
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 77.16862866609263
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 84.59956437228902,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 53.79869311686706
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 84.59956437228902
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 20.430023754833893,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -50,
        "score": 55.62504663395982
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 20.430023754833893
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 42.86130199817126,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 38.72883353074567
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 42.86130199817126
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 93.80428848041588,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 92.0890220248775
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 93.80428848041588
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.28120639560315,
      84.64714278257424
     ],
     "I": [
      50.218617607373794,
      58.73980484407819
     ],
     "capability": [
      64.3485197527183,
      71.10803595803117
     ],
     "composite": {
      "A": [
       44.392703292458805,
       46.89203249517508
      ],
      "B": [
       43.17507195851998,
       46.534331841823644
      ],
      "C": [
       31.09870948973084,
       44.91059204066695
      ]
     },
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.82393852673533,
     "capability": 68.39271102371643,
     "intelligence": 54.96148352069754
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7186666666666667,
      "calibration": 81.79703955151288,
      "cc": 63.661761610440315,
      "cc_by_type": {
       "choice": 63.661761610440315
      },
      "cc_tiered": 62.570157201540155,
      "cc_tiered_open": 62.113835115843074,
      "cc_tiered_sealed": 63.02647928723724,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.661761610440315
     },
     "noul": {
      "accuracy": 0.7946666666666666,
      "calibration": 80.88511475193768,
      "cc": 58.93333333333332,
      "cc_by_type": {
       "noul": 58.93333333333332
      },
      "cc_tiered": 63.34309233547301,
      "cc_tiered_open": 71.3768115942029,
      "cc_tiered_sealed": 55.309373076743114,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 58.93333333333332
     },
     "score": {
      "accuracy": 0.5466666666666666,
      "calibration": 82.78966127675545,
      "cc": 36.46304903717563,
      "cc_by_type": {
       "score": 36.46304903717563
      },
      "cc_tiered": 38.971201025079466,
      "cc_tiered_open": 43.862911895878305,
      "cc_tiered_sealed": 34.079490154280634,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.46304903717563
     }
    },
    "raw_on_A": {
     "calibration": 81.0046715261283,
     "capability": 66.7025205730405,
     "intelligence": 52.400369619952706
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 42.095287401061526,
    "Emphasis on Accuracy 60:20:20": 42.2473928103636,
    "Emphasis on Cost 20:20:60": 46.3627340067867,
    "Emphasis on Speed 20:60:20": 38.42054978162839,
    "Intelligence only": 42.477623199386635
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "adimyth",
   "axes": {
    "calibration": 83.26568672143415,
    "cost": 57.598862966277466,
    "intelligence": 28.917433770533798,
    "speed": 83.0508382606611
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08397997501761144,
      "ece_score": 83.2040049964777,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 83.2040049964777,
      "score_before_time_limit": 83.2040049964777,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.7466666666666666,
      "brier": 0.1702847698640915,
      "ece": 0.10141246114807827,
      "n": 375,
      "score": 79.71750777038434,
      "score_before_time_limit": 79.71750777038434,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.5966666666666666,
      "n": 375,
      "nrps": 0.12226058004692472,
      "score": 86.87554739744039,
      "score_before_time_limit": 86.87554739744039,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07011423600213387
     }
    },
    "score": 83.26568672143415
   },
   "capability": 56.09156024598397,
   "composite_ci95": {
    "A": [
     12.444898740601916,
     22.038300683254842
    ],
    "B": [
     10.440830481540472,
     19.26046301163513
    ],
    "C": [
     8.642290792084664,
     15.304375474482528
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the author's server: input_tokens counts EVERY token the backbone processed including the per-candidate verification branches (2.47M over the full draw, higher than one pass by construction); output_tokens 0, nothing generated. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.025904 per 1,000 decisions (0.401x the official cost cap USD 0.064595), against USD 0.049336 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04933552
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025904262626262626
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Compass 0.2.0 (adimyth, Qwen3.5-4B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's compass.server: per-candidate verification branches, readout fusion 0.5, debias 0.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 31.733302625698396,
    "I_sealed": 26.1015649153692,
    "base": 28.917433770533798,
    "excess": -4.2077890600265135,
    "gap": 5.6317377103291975,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 62.41376749591724,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 64.1396665618119,
       "judge": 60.44825313117996,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -4.347826086956519,
       "judge": -80.0,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 32.78614038117795,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.20061237036324,
       "hard": 36.58381707085273,
       "judge": 21.473132873563948,
       "standard": 32.953062268656744
      }
     },
     "sealed|choice": {
      "cc": 58.085681342882154,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 55.24459350063606,
       "judge": 54.162585700293846,
       "standard": 61.08415946002954
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -27.586206896551722,
       "hard": -47.27272727272728,
       "judge": -84.61538461538461,
       "standard": -55.223880597014926
      }
     },
     "sealed|score": {
      "cc": 20.21901340322545,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 39.1090569064439,
       "hard": 15.992779208880192,
       "judge": 13.206706256149614,
       "standard": 29.74492076092049
      }
     }
    }
   },
   "jevbench_score": 17.606155981656478,
   "key": "compass-0.2.0",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "adimyth/compass-lora-v2@bfa8af07916df491fde171d1da0d6d761304f2f3",
   "noul_decisive": {
    "acc_among_decisive": 0.968421052631579,
    "decisive": 95,
    "decisive_rate": 0.25333333333333335,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.26568672143415,
     "cost": 57.598862966277466,
     "intelligence": 28.917433770533798,
     "speed": 83.28951948101809
    },
    "board_release": "v1.6.43",
    "capability": 56.09156024598397,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 79,
    "capability_rank": 105,
    "composite_ci95": {
     "A": [
      12.450188556707262,
      22.048694801053795
     ],
     "B": [
      10.44380870456473,
      19.266813296946893
     ],
     "C": [
      8.645964275491151,
      15.311593611842914
     ]
    },
    "jevbench_score": 17.614153784977713,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 108,
    "rank": 144,
    "ranks": {
     "A": 144,
     "B": 148,
     "C": 147,
     "capability": 160
    },
    "scores": {
     "A": 17.614153784977713,
     "B": 15.129697598253454,
     "C": 12.23205123956786
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.6588797099888325,
     "p50_s_raw": 0.25443985499441624,
     "p95_s_adjusted": 0.7116097962949425,
     "p95_s_raw": 0.2808048981474712
    },
    "views": {
     "Balanced 33:33:33": 15.691595815525753,
     "Emphasis on Accuracy 60:20:20": 12.564181595428897,
     "Emphasis on Cost 20:20:60": 16.94946722349077,
     "Emphasis on Speed 20:60:20": 19.013249270056317,
     "Intelligence only": 9.672511174017334
    }
   },
   "rank": 163,
   "ranked": true,
   "ranks": {
    "A": 163,
    "B": 167,
    "C": 167,
    "capability": 178
   },
   "repo": "https://huggingface.co/adimyth/compass-lora-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.606155981656478,
    "B": 15.12497632290874,
    "C": 12.226497209483666
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6632696912623942,
    "p50_s_raw": 0.2566348456311971,
    "p95_s_adjusted": 0.7468373369425535,
    "p95_s_raw": 0.29841866847127674
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 25.16281616679272,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 0.0,
        "score": 17.15511516704482
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 25.16281616679272
      },
      "calibration_gen": {
       "accuracy": 0.36633663366336633,
       "cc": -10.700517157662324,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -60.0,
        "score": 1.0691802343301027
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.44554455445544555,
       "cc": -3.1609446697062276,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -70.37037037037037,
        "score": -6.592138435496286
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 77.56082309299116,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 72.68246927897344
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.56082309299116
      },
      "jaggedness": {
       "accuracy": 0.4166666666666667,
       "cc": -7.521417581149862,
       "cc_by_type": {
        "choice": 43.69586140519731,
        "noul": -72.97297297297297,
        "score": 6.712858824326073
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.5591054313099042,
       "cc": 18.55523558036446,
       "cc_by_type": {
        "choice": 68.3014880690323,
        "noul": -54.639175257731964,
        "score": 42.00339392979304
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 18.55523558036446
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -25.04602397371077,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": -100.0,
        "score": 2.2812829175773763
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -33.53428485635812,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100.0,
        "score": -14.004916424744462
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5745454545454546,
       "cc": 20.339012444839206,
       "cc_by_type": {
        "choice": 67.72089990218454,
        "noul": -41.666666666666664,
        "score": 34.96280409899973
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 20.339012444839206
      },
      "negation_pair_gen": {
       "accuracy": 0.37209302325581395,
       "cc": -25.591151403349343,
       "cc_by_type": {
        "choice": 53.333333333333336,
        "noul": -100.0,
        "score": -30.106787543381365
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.75,
       "cc": 68.04260985352863,
       "cc_by_type": {
        "choice": 68.04260985352863
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 68.04260985352863
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 79.40020031093738,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100.0,
        "score": 62.22462495683616
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.40020031093738
      },
      "usecase": {
       "accuracy": 0.4772727272727273,
       "cc": 9.25295253481841,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": -54.16666666666667,
        "score": 26.043171329945412
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 9.25295253481841
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -22.487984477627545,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50.0,
        "score": -41.27347724240644
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": -7.328512183344543,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -100.0,
        "score": 19.48911782784656
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 34.097952211043115,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": -25.0,
        "score": 35.25845840304084
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 34.097952211043115
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 36.12028010447652,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 91.97350325570144
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 36.12028010447652
      },
      "en": {
       "accuracy": 0.5125977410947002,
       "cc": 9.510128503054053,
       "cc_by_type": {
        "choice": 57.60364164459792,
        "noul": -51.798561151079134,
        "score": 22.72530501564337
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 9.510128503054053
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 44.557456303481054,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 25.0,
        "score": 8.672368910443161
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 44.557456303481054
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -22.77211870429048,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": 30.04429962483347
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 9.884178877858309,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": -100.0,
        "score": 50.82900722181021
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 9.884178877858309
      },
      "hi": {
       "accuracy": 0.2857142857142857,
       "cc": -23.991596849230035,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100.0,
        "score": -21.695460938751566
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.6666666666666666,
       "cc": 21.774974165868226,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": -33.333333333333336,
        "score": 12.68787391355688
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 21.774974165868226
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 16.651900813212105,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": -40.0,
        "score": 32.846885918587866
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 16.651900813212105
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -1.514851221719643,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -4.544553665158935
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.9726455172006,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.5846032182685
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.9726455172006
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 4.0277366115052375,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 38.05723580854167
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 4.0277366115052375
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5384615384615384,
       "cc": 15.744381156105492,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -60.0,
        "score": 30.085449858498457
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 15.744381156105492
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 25.49445136170787,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 49.51812132142296
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 25.49445136170787
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -20.332935033810834,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100.0,
        "score": 41.12885447303558
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 36.968154265366756,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 10.904462796100267
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 36.968154265366756
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 49.387703797391886,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 12.679240424433747
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 49.387703797391886
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -3.515405794221941,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -26.365426422213844
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6893333333333334,
       "cc": 59.872940546126024,
       "cc_by_type": {
        "choice": 59.872940546126024
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.872940546126024
      },
      "noul": {
       "accuracy": 0.24533333333333332,
       "cc": -50.933333333333344,
       "cc_by_type": {
        "noul": -50.933333333333344
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.4666666666666667,
       "cc": 24.379746613482347,
       "cc_by_type": {
        "score": 24.379746613482347
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 24.379746613482347
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -27.575892260550187,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -100.0,
        "score": -34.727676781650565
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.20833333333333334,
       "cc": -33.860695446041085,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -100.0,
        "score": -27.04792484744003
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 35.210411379905146,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0.0,
        "score": 38.15155934296747
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 35.210411379905146
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -18.69170604393453,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": 14.100320464687643
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 10.857814798771154,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 23.87779222240042
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 10.857814798771154
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -38.8677190566742,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -100.0,
        "score": -53.71007540901631
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": 1.1070274802771773,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": 23.769261712540214
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 1.1070274802771773
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -14.364855425973063,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 30.076165429397882
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 18.20005544196467,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60.0,
        "score": 35.43349965922733
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.20005544196467
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 0.6193288792792195,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 12.279114403719172
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.6193288792792195
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 21.46736464644896,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 9.363334249424426
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 21.46736464644896
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -19.884112566167715,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 4.863791333754919
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 14.995552254048038,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -60.0,
        "score": 39.53560109995602
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 14.995552254048038
      },
      "model_routing": {
       "accuracy": 0.5576036866359447,
       "cc": 13.046336664137272,
       "cc_by_type": {
        "choice": 77.7922077922078,
        "noul": -66.66666666666667,
        "score": 28.01346886687068
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 13.046336664137272
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -0.6512459807687018,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": 55.627835954047065
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 51.828105924296004,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 38.86407636156023
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 51.828105924296004
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -17.472858265243993,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -100.0,
        "score": 6.694725696878856
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": -3.514125054614875,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": -100.0,
        "score": 18.443132082532188
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 70.03763968705526,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50.0,
        "score": 70.78907564479567
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 70.03763968705526
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.62672822912583,
      84.7616620699159
     ],
     "I": [
      25.13342857907104,
      31.751819803526313
     ],
     "capability": [
      52.68979192721088,
      57.180429826061115
     ],
     "composite": {
      "A": [
       12.444898740601916,
       22.038300683254842
      ],
      "B": [
       10.440830481540472,
       19.26046301163513
      ],
      "C": [
       8.642290792084664,
       15.304375474482528
      ]
     },
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.26568672143415,
     "capability": 56.09156024598397,
     "intelligence": 28.917433770533798
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6893333333333334,
      "calibration": 83.2040049964777,
      "cc": 59.872940546126024,
      "cc_by_type": {
       "choice": 59.872940546126024
      },
      "cc_tiered": 60.2497244193997,
      "cc_tiered_open": 62.41376749591724,
      "cc_tiered_sealed": 58.085681342882154,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.872940546126024
     },
     "noul": {
      "accuracy": 0.24533333333333332,
      "calibration": 79.71750777038434,
      "cc": -50.933333333333344,
      "cc_by_type": {
       "noul": -50.933333333333344
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.4666666666666667,
      "calibration": 86.87554739744039,
      "cc": 24.379746613482347,
      "cc_by_type": {
       "score": 24.379746613482347
      },
      "cc_tiered": 26.502576892201702,
      "cc_tiered_open": 32.78614038117795,
      "cc_tiered_sealed": 20.21901340322545,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 24.379746613482347
     }
    },
    "raw_on_A": {
     "calibration": 79.36406944543388,
     "capability": 54.5556229058179,
     "intelligence": 29.747176366201913
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.683133619464702,
    "Emphasis on Accuracy 60:20:20": 12.5609255517335,
    "Emphasis on Cost 20:20:60": 16.943542137787777,
    "Emphasis on Speed 20:60:20": 18.990900299235857,
    "Intelligence only": 9.672511174017334
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 87.47631981746561,
    "cost": 48.45008367155653,
    "intelligence": 35.82270435669186,
    "speed": 89.53277608263193
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05090298449957041,
      "ece_score": 89.81940310008592,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.81940310008592,
      "score_before_time_limit": 89.81940310008592,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.46648793565683644,
      "brier": 0.14462735182051856,
      "ece": 0.08496977140192581,
      "n": 373,
      "score": 83.00604571961485,
      "score_before_time_limit": 83.00604571961485,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.5766129032258064,
      "n": 372,
      "nrps": 0.09926110585728637,
      "score": 89.60351063269607,
      "score_before_time_limit": 89.60351063269607,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.054334340744396026
     }
    },
    "score": 87.47631981746561
   },
   "capability": 61.64951208707873,
   "composite_ci95": {
    "A": [
     20.38789758812859,
     34.454740526903954
    ],
    "B": [
     17.998378420380163,
     31.53436818469049
    ],
    "C": [
     14.158262213978189,
     23.92690314368331
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: lineage base Qwen/Qwen3.5-9B (the card's base vllm-sr/Decision-1.0-Lux-9B is itself a fine-tune and RAISES in the module; Qwen3.5-9B lineage verified by config comparison in the earlier v1.5 measurement) -> qwen/qwen3.5-9b USD 0.09999999999999999/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835, the module this row was first priced with (the later re-freezes only add hosted-API launch-list keys), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). Tokens as reported by the author's runtime (output_tokens 0, nothing generated). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.052279 per 1,000 decisions (0.809x the official cost cap USD 0.064595), against USD 0.051757 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0517566
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05227939393939394
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 2.0 Lux 9B (vLLM-SR, Qwen3.5-9B lineage)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20150, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Authors' own in-process runtime behind the evaluator's loopback shim (verbatim from the earlier v1.5 measurement), serial requests.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.41506937866855,
    "I_sealed": 30.230339334715172,
    "base": 35.82270435669186,
    "excess": 1.3452032735976651,
    "gap": 11.184730043953376,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 65.97342471411105,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 64.1396665618119,
       "judge": 72.31377719182598,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 10.641821946169772,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -4.347826086956519,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 47.62996147572483,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.01722460619536,
       "hard": 42.15889431879225,
       "judge": 48.06315937576442,
       "standard": 53.22866737429535
      }
     },
     "sealed|choice": {
      "cc": 59.49041617562122,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 53.16294668671215,
       "judge": 55.92556317335946,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": -9.090909090909093,
       "judge": -41.53846153846153,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 31.200601828524306,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 52.52957611300084,
       "hard": 27.31494795658824,
       "judge": 23.847538197285967,
       "standard": 39.33701787701567
      }
     }
    }
   },
   "jevbench_score": 27.094140204147248,
   "key": "decision2-lux-9b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/Decision-2.0-Lux-9B@78bf3c03d9147aeb30b641edfe0e30ed04887ca5",
   "noul_decisive": {
    "acc_among_decisive": 0.9447236180904522,
    "decisive": 199,
    "decisive_rate": 0.5335120643431636,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.47631981746561,
     "cost": 48.45008367155653,
     "intelligence": 35.82270435669186,
     "speed": 89.93254869015362
    },
    "board_release": "v1.6.43",
    "capability": 61.64951208707873,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 56,
    "capability_rank": 81,
    "composite_ci95": {
     "A": [
      20.401360460654523,
      34.47961583493885
     ],
     "B": [
      18.006770417598776,
      31.551034699005438
     ],
     "C": [
      14.167611431010082,
      23.944177663151983
     ]
    },
    "jevbench_score": 27.113058507774674,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 75,
    "rank": 108,
    "ranks": {
     "A": 108,
     "B": 108,
     "C": 120,
     "capability": 125
    },
    "scores": {
     "A": 27.113058507774674,
     "B": 24.336976546681964,
     "C": 18.828512852621298
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2866953719989397,
     "p50_s_raw": 0.06834768599946983,
     "p95_s_adjusted": 0.3542619008774636,
     "p95_s_raw": 0.10213095043873181
    },
    "views": {
     "Balanced 33:33:33": 24.230269361637582,
     "Emphasis on Accuracy 60:20:20": 20.8638737710436,
     "Emphasis on Cost 20:20:60": 23.871066823962767,
     "Emphasis on Speed 20:60:20": 29.419879270169634,
     "Intelligence only": 17.265698468557986
    }
   },
   "rank": 122,
   "ranked": true,
   "ranks": {
    "A": 122,
    "B": 122,
    "C": 137,
    "capability": 142
   },
   "repo": "https://huggingface.co/vllm-sr/Decision-2.0-Lux-9B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 27.094140204147248,
    "B": 24.324780085455362,
    "C": 18.815375141768918
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2906017030240037,
    "p50_s_raw": 0.07030085151200183,
    "p95_s_adjusted": 0.3831988849298795,
    "p95_s_raw": 0.11659944246493978
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 65.38134362243356,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 29.477364200633982
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 65.38134362243356
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 4.099569847308422,
       "cc_by_type": {
        "choice": 30.010604453870634,
        "noul": -28.000000000000004,
        "score": 10.288105088054634
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 4.099569847308422
      },
      "composite_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 26.33004954263009,
       "cc_by_type": {
        "choice": 77.23577235772358,
        "noul": -48.14814814814815,
        "score": 49.90252441831484
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 26.33004954263009
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 62.654116824285275,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 67.96235047285583
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 62.654116824285275
      },
      "jaggedness": {
       "accuracy": 0.4,
       "cc": -3.3916129517076663,
       "cc_by_type": {
        "choice": 30.702598652550538,
        "noul": -56.75675675675676,
        "score": 15.879319249083224
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.8146964856230032,
       "cc": 67.14156045601494,
       "cc_by_type": {
        "choice": 77.01857885004843,
        "noul": 67.01030927835052,
        "score": 57.395793239645876
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 67.14156045601494
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -11.681952585069832,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -100.0,
        "score": 14.43867832726472
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6363636363636364,
       "cc": 33.066005227920634,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": -11.111111111111116,
        "score": 38.67560869901385
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 33.066005227920634
      },
      "negation_pair_gen": {
       "accuracy": 0.4883720930232558,
       "cc": 2.67222418279357,
       "cc_by_type": {
        "choice": 56.66666666666668,
        "noul": -60.0,
        "score": 11.350005881714031
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 2.67222418279357
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 74.77426699693255,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 80.0,
        "score": 71.3498280178247
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.77426699693255
      },
      "usecase": {
       "accuracy": 0.509090909090909,
       "cc": 26.66485457151172,
       "cc_by_type": {
        "choice": 47.30392156862747,
        "noul": -4.1666666666666625,
        "score": 36.85730881257436
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 26.66485457151172
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 9.056945780385973,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -15.686305515984934
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 9.056945780385973
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 17.326598840370945,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 26.787784232326363
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 17.326598840370945
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 67.11320598246611,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 50.233423257132834
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 67.11320598246611
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 33.09357426464621,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 99.46694253323099
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 33.09357426464621
      },
      "en": {
       "accuracy": 0.6046915725456126,
       "cc": 31.4531064905039,
       "cc_by_type": {
        "choice": 58.701992897328545,
        "noul": 0.0,
        "score": 35.657326574183145
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.4531064905039
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 63.51130608648939,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 10.10977470318759
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 63.51130608648939
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 19.74093935418468,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 25.34303664178902
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 19.74093935418468
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 40.99009270088603,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 65.32321927912866
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 40.99009270088603
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 18.916283367574735,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50.0,
        "score": 23.508626639037043
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 18.916283367574735
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 74.61768112255443,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 23.853043367663275
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 74.61768112255443
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 14.22770772278479,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -19.999999999999996,
        "score": 34.16842896660695
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 14.22770772278479
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -9.761328844855484,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 15.160457909878
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.09322796696891,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 82.9463505675734
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.09322796696891
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 9.435501837148886,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 28.306505511446655
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.435501837148886
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 39.71775347984494,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": -19.999999999999996,
        "score": 46.77069590292883
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 39.71775347984494
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 66.8390486833019,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 42.94495983894516
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 66.8390486833019
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 3.6562373029894957,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 47.564456589819535
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 3.6562373029894957
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 24.52332791887444,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50.0,
        "score": -54.69088580859407
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 24.52332791887444
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.66005480942651,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 41.23822894440856
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.66005480942651
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 6.234913945334711,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -47.961924830662525
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 6.234913945334711
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7026666666666667,
       "cc": 61.595131938996154,
       "cc_by_type": {
        "choice": 61.595131938996154
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.595131938996154
      },
      "noul": {
       "accuracy": 0.5013333333333333,
       "cc": 0.2666666666666595,
       "cc_by_type": {
        "noul": 0.2666666666666595
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.2666666666666595
      },
      "score": {
       "accuracy": 0.5573333333333333,
       "cc": 35.76324809476839,
       "cc_by_type": {
        "score": 35.76324809476839
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.76324809476839
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -21.716999577358354,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -100.0,
        "score": -17.150998732075063
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 30.171019428261456,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 11.111111111111116,
        "score": 16.669027919014855
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 30.171019428261456
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 66.97464414728488,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 50.923932441854646
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 66.97464414728488
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -15.449793509057187,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": -19.44879573184992
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 23.250864501749692,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 32.071434084959215
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 23.250864501749692
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -65.29273959291613,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -66.66666666666667,
        "score": -141.16123764667293
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 21.627541374440955,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 37.15153168634807
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.627541374440955
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": 10.26483205645294,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50.0,
        "score": 53.96522787667589
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 10.26483205645294
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 9.522250670781048,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 51.066752012343144
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 9.522250670781048
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 21.435076662665292,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 0.0,
        "score": 28.545058681786028
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 21.435076662665292
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 33.301792034961935,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 22.644394192741114
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 33.301792034961935
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -31.99789804370859,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -33.62810273327632
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 25.452132963671094,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 30.90534322882519
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 25.452132963671094
      },
      "model_routing": {
       "accuracy": 0.8341013824884793,
       "cc": 67.04627110748747,
       "cc_by_type": {
        "choice": 84.80519480519482,
        "noul": 72.72727272727273,
        "score": 43.60634578999484
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.04627110748747
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 6.899543267261227,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 66.76388891886621
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 6.899543267261227
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 50.511962779197546,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 51.62873884827046
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.511962779197546
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 1.2750601937910584,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": -16.618169172321416
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 1.2750601937910584
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 12.33326926457949,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 42.79690924301383
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 12.33326926457949
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 91.439262104532,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 84.99394289722593
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 91.439262104532
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.99931384938026,
      87.44256636228893
     ],
     "I": [
      31.9535738870049,
      39.72745829130512
     ],
     "capability": [
      57.416494415371886,
      62.68561954122301
     ],
     "composite": {
      "A": [
       20.38789758812859,
       34.454740526903954
      ],
      "B": [
       17.998378420380163,
       31.53436818469049
      ],
      "C": [
       14.158262213978189,
       23.92690314368331
      ]
     },
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.47631981746561,
     "capability": 61.64951208707873,
     "intelligence": 35.82270435669186
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7026666666666667,
      "calibration": 89.81940310008592,
      "cc": 61.595131938996154,
      "cc_by_type": {
       "choice": 61.595131938996154
      },
      "cc_tiered": 62.731920444866134,
      "cc_tiered_open": 65.97342471411105,
      "cc_tiered_sealed": 59.49041617562122,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.595131938996154
     },
     "noul": {
      "accuracy": 0.5013333333333333,
      "calibration": 83.00604571961485,
      "cc": 0.2666666666666595,
      "cc_by_type": {
       "noul": 0.2666666666666595
      },
      "cc_tiered": 5.320910973084886,
      "cc_tiered_open": 10.641821946169772,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.2666666666666595
     },
     "score": {
      "accuracy": 0.5573333333333333,
      "calibration": 89.60351063269607,
      "cc": 35.76324809476839,
      "cc_by_type": {
       "score": 35.76324809476839
      },
      "cc_tiered": 39.41528165212457,
      "cc_tiered_open": 47.62996147572483,
      "cc_tiered_sealed": 31.200601828524306,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.76324809476839
     }
    },
    "raw_on_A": {
     "calibration": 81.01141299701578,
     "capability": 59.06449579734621,
     "intelligence": 37.11757859767665
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 24.21012644558836,
    "Emphasis on Accuracy 60:20:20": 20.85490936444613,
    "Emphasis on Cost 20:20:60": 23.85933276095094,
    "Emphasis on Speed 20:60:20": 29.36648036746943,
    "Intelligence only": 17.265698468557986
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Nace AI",
   "axes": {
    "calibration": 89.56795021085651,
    "cost": 55.336763284582396,
    "intelligence": 78.89021516142793,
    "speed": 89.64251650815261
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0614702656186141,
      "ece_score": 87.70594687627718,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.70594687627718,
      "score_before_time_limit": 87.70594687627718,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0668449197860963,
      "brier": 0.06226407500935928,
      "ece": 0.05292380823378494,
      "n": 374,
      "score": 89.41523835324301,
      "score_before_time_limit": 89.41523835324301,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.946524064171123,
      "interval90_width": 0.13235294117647062,
      "n": 374,
      "nrps": 0.04098278578392381,
      "score": 91.58266540304936,
      "score_before_time_limit": 91.58266540304936,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06368195307754448
     }
    },
    "score": 89.56795021085651
   },
   "capability": 84.22908268614222,
   "composite_ci95": {
    "A": [
     73.86656464052277,
     76.21494021691677
    ],
    "B": [
     74.05703618344492,
     77.33666212419148
    ],
    "C": [
     73.86656464052277,
     76.21494021691677
    ]
   },
   "cost": {
    "basis": "Launch list price: Drex Max public launch list tariff USD 0.06 per million input tokens, output not billed, https://drex.nace.ai/docs/drex/reference/pricing (saved copy sha256 95d1c429, fetched 21 Oct 2026 20:18 UTC; fetched again 22 Oct 2026 01:12 UTC with identical bytes). Counted as an operator standard launch list price by the frozen pricing module pricing_v15 (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7), which now carries a launch-list-price line for this product. Applied to provider-reported usage; not a base-model estimate. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 762,693 input / 262,045 output tokens. All-item usage 1,942,633 input / 264,615 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.07770532
    },
    "kind": "listed",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.030815878787878788
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Drex Max (Nace AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane; full S1200+P300 draw v1.6-regular-20261012, unchanged evaluator API runner from Sandy, customer API key; full-set exposure gate logged all sealed items before the first scored request. Native probabilities; latency has no API adjustment.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 81.66095761967301,
    "I_sealed": 76.11947270318284,
    "base": 78.89021516142793,
    "excess": -4.298041853865534,
    "gap": 5.541484916490177,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 85.06220262367293,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 90.56307014784524,
       "judge": 84.17930125247199,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 85.29951690821257,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 70.0,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 74.62115332713357,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.894225726887605,
       "hard": 77.86140310269487,
       "judge": 79.87410650396917,
       "standard": 67.12468781088054
      }
     },
     "sealed|choice": {
      "cc": 83.42288824648435,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 80.2243552677229,
       "judge": 80.60724779627816,
       "standard": 87.66083104830204
      }
     },
     "sealed|noul": {
      "cc": 70.73090779524131,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 60.00000000000001,
       "judge": 72.30769230769232,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 74.20462206782285,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 83.85760877630031,
       "hard": 76.79135886109052,
       "judge": 72.01874094079821,
       "standard": 67.48347681758575
      }
     }
    }
   },
   "jevbench_score": 75.37621071625281,
   "key": "drex-max",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary vendor-hosted API; weights and serving code not independently inspected",
   "listing": "ranked",
   "measured_in": "v1.6.38",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://drex.nace.ai#drex-max",
   "noul_decisive": {
    "acc_among_decisive": 0.9426934097421203,
    "decisive": 349,
    "decisive_rate": 0.9331550802139037,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 6,
    "axes": {
     "calibration": 89.56795021085651,
     "cost": 55.336763284582396,
     "intelligence": 78.89021516142793,
     "speed": 89.80651018227358
    },
    "board_release": "v1.6.43",
    "capability": 84.22908268614222,
    "capability_api_board_rank": 3,
    "capability_open_board_rank": null,
    "capability_rank": 5,
    "composite_ci95": {
     "A": [
      73.89436205802828,
      76.24453356276406
     ],
     "B": [
      74.07938728544397,
      77.3610370295492
     ],
     "C": [
      73.89436205802828,
      76.24453356276406
     ]
    },
    "jevbench_score": 75.40515618514833,
    "measured_in": "v1.6.38",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 7,
    "ranks": {
     "A": 7,
     "B": 3,
     "C": 7,
     "capability": 10
    },
    "scores": {
     "A": 75.40515618514833,
     "B": 76.0773154141038,
     "C": 75.40515618514833
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.2520033140317537,
     "p50_s_raw": 0.2520033140317537,
     "p95_s_adjusted": 0.41489932631957344,
     "p95_s_raw": 0.41489932631957344
    },
    "views": {
     "Balanced 33:33:33": 71.62971034982631,
     "Emphasis on Accuracy 60:20:20": 74.3674134322497,
     "Emphasis on Cost 20:20:60": 64.08251746798757,
     "Emphasis on Speed 20:60:20": 77.93969491847153,
     "Intelligence only": 78.89021516142793
    }
   },
   "rank": 7,
   "ranked": true,
   "ranks": {
    "A": 7,
    "B": 4,
    "C": 7,
    "capability": 13
   },
   "repo": "https://drex.nace.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 75.37621071625281,
    "B": 76.05374262335641,
    "C": 75.37621071625281
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2538145645521581,
    "p50_s_raw": 0.2538145645521581,
    "p95_s_adjusted": 0.42779116374440485,
    "p95_s_raw": 0.42779116374440485
   },
   "status": {
    "answered_ok": 1498,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API through the unchanged TypeSafe adapter; no vendor code ran locally. 2 HTTP 529 provider-overloaded failures retained and scored as failures. Public JevBench items were used to pick the checkpoint; the public set is treated as seen during selection. Customer statements about serving and training are unverified.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 87.32730011427283,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 70.31523367615182
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.32730011427283
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 66.4168117636221,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 60.00000000000001,
        "score": 74.245133063931
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 66.4168117636221
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 90.98786119357764,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 83.62302350846554
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.98786119357764
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 99.9811260569162,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.94337817074862
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.9811260569162
      },
      "jaggedness": {
       "accuracy": 0.7611111111111111,
       "cc": 62.67781187887625,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": 45.945945945945944,
        "score": 78.17988622581275
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 62.67781187887625
      },
      "llm_router": {
       "accuracy": 0.9265175718849841,
       "cc": 83.03900487815365,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 93.81443298969072,
        "score": 60.84982123268957
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.03900487815365
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": -7.819371042052626,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": 0.0,
        "score": -84.74843570680304
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 67.45921990260773,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 50.0,
        "score": 77.11992774906025
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 67.45921990260773
      },
      "multilingual": {
       "accuracy": 0.8436363636363636,
       "cc": 76.08800384521719,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 72.22222222222223,
        "score": 74.6267257333837
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 76.08800384521719
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 74.73255877300765,
       "cc_by_type": {
        "choice": 96.66666666666667,
        "noul": 66.66666666666667,
        "score": 60.86434298568961
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.73255877300765
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 83.67877565230252,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 66.0513419719226
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.67877565230252
      },
      "usecase": {
       "accuracy": 0.8454545454545455,
       "cc": 73.95953326890493,
       "cc_by_type": {
        "choice": 82.84313725490196,
        "noul": 62.5,
        "score": 76.53546255181287
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 73.95953326890493
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 69.91186881655202,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": 78.7832254972751
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 69.91186881655202
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 58.44437051865251,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 41.99977822262423
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 58.44437051865251
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.69742394292284,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 86.98607713850303
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.69742394292284
      },
      "el": {
       "accuracy": 0.9333333333333333,
       "cc": 94.41307136986232,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100.0,
        "score": 99.9989906458998
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 94.41307136986232
      },
      "en": {
       "accuracy": 0.8705473501303215,
       "cc": 78.87188263043777,
       "cc_by_type": {
        "choice": 85.28209321340964,
        "noul": 75.53956834532374,
        "score": 75.7939863325799
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 78.87188263043777
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 72.33983928560271,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 26.80744607866782
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 72.33983928560271
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.76471453815516,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.96081028113215
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.76471453815516
      },
      "fr": {
       "accuracy": 1.0,
       "cc": 99.0532296631748,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.15968898952437
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 99.0532296631748
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 70.61814713883847,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75.0,
        "score": 53.61421795282821
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.61814713883847
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 75.83799074906126,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 41.54359032980268
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 75.83799074906126
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 54.328167608501246,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 39.99999999999999,
        "score": 22.98450282550375
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 54.328167608501246
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 92.58033340509049,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 77.74100021527148
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 92.58033340509049
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.7683454076434,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.9717028895969
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.7683454076434
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 17.73905043083302,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": -28.168130092782317
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 17.73905043083302
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8717948717948718,
       "cc": 81.50650313356748,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 79.7543803274904
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 81.50650313356748
      },
      "pt": {
       "accuracy": 0.8928571428571429,
       "cc": 84.51664635695569,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100.0,
        "score": 74.76384596538684
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 84.51664635695569
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 47.512402499217224,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 59.13295217850272
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 47.512402499217224
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 82.40769768899798,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 68.96222350177659
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 82.40769768899798
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.90750936589232,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 83.98059261380598
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.90750936589232
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 57.55187446448804,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 39.32229006013077
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 57.55187446448804
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8813333333333333,
       "cc": 84.67249660345587,
       "cc_by_type": {
        "choice": 84.67249660345587
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.67249660345587
      },
      "noul": {
       "accuracy": 0.8773333333333333,
       "cc": 75.46666666666665,
       "cc_by_type": {
        "noul": 75.46666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 75.46666666666665
      },
      "score": {
       "accuracy": 0.8186666666666667,
       "cc": 74.40265595252149,
       "cc_by_type": {
        "score": 74.40265595252149
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 74.40265595252149
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 40.94013002702126,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -25.0,
        "score": 63.82039008106377
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 40.94013002702126
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 94.74649038827987,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 84.23947116483957
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 94.74649038827987
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 56.18418746315232,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 100.0,
        "score": 33.59321279596102
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 56.18418746315232
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 10.678347318840489,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": -42.81875921306917
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 10.678347318840489
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 65.8121592702109,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 93.08865172367618
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 65.8121592702109
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 63.11469221028199,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": 81.16797600191516
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 63.11469221028199
      },
      "financial_crime": {
       "accuracy": 0.8666666666666667,
       "cc": 69.85403609350806,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 42.85714285714286,
        "score": 76.50888699200877
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 69.85403609350806
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.51172431161607,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50.0,
        "score": 68.89405795226982
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.51172431161607
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 58.5760942817511,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 78.22828284525329
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 58.5760942817511
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 78.42207904152188,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100.0,
        "score": 48.114202863323655
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 78.42207904152188
      },
      "lead_generation": {
       "accuracy": 0.88,
       "cc": 79.80323774108199,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 61.63193544546821
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 79.80323774108199
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 26.355654431569025,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 42.50782350976085
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 26.355654431569025
      },
      "llm_guardrails": {
       "accuracy": 1.0,
       "cc": 96.94611245850201,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 90.83833737550606
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 96.94611245850201
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 81.10660381485808,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100.0,
        "score": 49.16396728873009
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.10660381485808
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 84.70157923454258,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100.0,
        "score": 77.13736726216898
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 84.70157923454258
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 86.1084876352515,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": 75.0385548277601
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 86.1084876352515
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 51.609571747494265,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 50.0,
        "score": 19.6070403656355
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 51.609571747494265
      },
      "scientific_discovery": {
       "accuracy": 0.8695652173913043,
       "cc": 69.05169114747838,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 33.33333333333333,
        "score": 88.3144937322902
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 69.05169114747838
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.97852779958207,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.9355833987462
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.97852779958207
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.5532389091344,
      91.2233312473974
     ],
     "I": [
      74.71372483564313,
      82.2819224684923
     ],
     "capability": [
      80.68052980412993,
      86.222957448835
     ],
     "composite": {
      "A": [
       73.86656464052277,
       76.21494021691677
      ],
      "B": [
       74.05703618344492,
       77.33666212419148
      ],
      "C": [
       73.86656464052277,
       76.21494021691677
      ]
     },
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.56795021085651,
     "capability": 84.22908268614222,
     "intelligence": 78.89021516142793
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8813333333333333,
      "calibration": 87.70594687627718,
      "cc": 84.67249660345587,
      "cc_by_type": {
       "choice": 84.67249660345587
      },
      "cc_tiered": 84.24254543507864,
      "cc_tiered_open": 85.06220262367293,
      "cc_tiered_sealed": 83.42288824648435,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.67249660345587
     },
     "noul": {
      "accuracy": 0.8773333333333333,
      "calibration": 89.41523835324301,
      "cc": 75.46666666666665,
      "cc_by_type": {
       "noul": 75.46666666666665
      },
      "cc_tiered": 78.01521235172694,
      "cc_tiered_open": 85.29951690821257,
      "cc_tiered_sealed": 70.73090779524131,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 75.46666666666665
     },
     "score": {
      "accuracy": 0.8186666666666667,
      "calibration": 91.58266540304936,
      "cc": 74.40265595252149,
      "cc_by_type": {
       "score": 74.40265595252149
      },
      "cc_tiered": 74.41288769747821,
      "cc_tiered_open": 74.62115332713357,
      "cc_tiered_sealed": 74.20462206782285,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 74.40265595252149
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.59488791056847,
    "Emphasis on Accuracy 60:20:20": 74.3448882133178,
    "Emphasis on Cost 20:20:60": 64.06579112533402,
    "Emphasis on Speed 20:60:20": 77.86551943721958,
    "Intelligence only": 78.89021516142793
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Fastino",
   "axes": {
    "calibration": 92.84353248115252,
    "cost": 59.111125613009825,
    "intelligence": 76.06842865080436,
    "speed": 87.00413908296701
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04346043152131592,
      "ece_score": 91.30791369573681,
      "mean_tvd": null,
      "n": 738,
      "n_ece_hard": 304,
      "n_tvd": 0,
      "score": 91.30791369573681,
      "score_before_time_limit": 91.30791369573681,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.13746630727762799,
      "brier": 0.0529066276825842,
      "ece": 0.03815484610642815,
      "n": 371,
      "score": 92.12139262649261,
      "score_before_time_limit": 92.36903077871436,
      "time_limit_failed_share": 0.002680965147453083
     },
     "score": {
      "interval90_coverage": 0.9892183288409704,
      "interval90_width": 0.21428571428571425,
      "n": 371,
      "nrps": 0.0377969402276481,
      "score": 95.10129112122816,
      "score_before_time_limit": 95.35625436820196,
      "time_limit_failed_share": 0.00267379679144385,
      "top_ece": 0.02753898620415633
     }
    },
    "score": 92.84353248115252
   },
   "capability": 84.45598056597845,
   "composite_ci95": {
    "A": [
     74.90033635084451,
     77.08874894746072
    ],
    "B": [
     74.3293214235386,
     77.56901714437397
    ],
    "C": [
     74.90033635084451,
     77.08874894746072
    ]
   },
   "cost": {
    "basis": "Operator's own published standard usage tariff: USD 0.04 per 1M INPUT tokens, output not billed (docs.fastino.ai/pricing, GLiDE row; page read 22 Oct 2026 20:34 UTC, saved copy sha256 22c0758f). Counted under interpretation I-1 (a manufacturer's standard, non-promotional list price counts from day 1) by the frozen pricing module pricing_v15 (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7), which carries one added launch-list-price key for this row and nothing else. Applied to this row's own provider-reported input tokens on the answered rows; the 17 requests that returned an error are priced at the API's 65,536-token prompt ceiling, the documented context estimate. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common items of this draw, leaving out the 15 very long items that count for no row; this row scored 1,497 items, so the basis here is 1,482 items. This row: USD 0.023065 per 1,000 decisions (0.357x the official cost cap USD 0.064595), against USD 0.049101 over all 1,497 scored items. The older measurement of this vendor's GLiDE on 6 Oct 2026 (key fastino-glide) was priced at the then tariff of USD 0.15 per 1M input tokens and is unchanged.",
    "common_basis": {
     "n_items": 1482,
     "rule": "common",
     "usd_per_1000_all_items": 0.0491012959251837
    },
    "kind": "mixed",
    "usage_estimated_rows": 17,
    "usd_per_1000": 0.023065479082321188
   },
   "custody_exclusion": {
    "derived_gold_sha256": "9f1f8200d0a1b4384f43ffc0a91f3f36e4790fc3dbcef7486bf233b0142baa47",
    "exclusion_list_sha256": "31b47af58dce134a70e5cbf60aad40db0000dd398f26649070fb7c9dfd1deeb3",
    "items": 3,
    "rule": "an item a provider has already seen is never scored for that provider; the 3 items were never sent and are not in this row's gold",
    "scored_items": 1497
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1497,
    "treatment": "share"
   },
   "display": "Fastino GLiDE (hosted API, 22 Oct 2026)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.fastino.ai /v1/systemone API, model `fastino/GLiDE` (responses report glide_v2), evaluator key, every request with store:false; latency measured from Sandy (Germany) on every item without API adjustment. Full S+P draw of v1.6-regular-20261012 minus the 3 custody-excluded items (1,497 requests, one pass, no retries) through the unchanged run_api_full_v161.py; the full-set exposure gate logged the 1,197 sealed items to the provider before the first request. Measured 22 Oct 2026 by the evaluator; the vendor did not take part in the run.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.78345749455909,
    "I_sealed": 72.35339980704964,
    "base": 76.06842865080436,
    "excess": -2.4094690828462646,
    "gap": 7.430057687509446,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 90.51495356571994,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 86.78829820698331,
       "judge": 96.044825313118,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 75.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 70.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 72.88717875232587,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.7719731384208,
       "hard": 72.51880498670339,
       "judge": 82.85006684010338,
       "standard": 59.2371969588571
      }
     },
     "sealed|choice": {
      "cc": 86.92305256837759,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 81.78559037816582,
       "judge": 87.65915768854063,
       "standard": 90.50833157561695
      }
     },
     "sealed|noul": {
      "cc": 62.1766990537663,
      "n": {
       "easy": 58,
       "hard": 109,
       "judge": 64,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 50.458715596330286,
       "judge": 65.625,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 67.96044779900504,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 66,
       "standard": 75
      },
      "tiers": {
       "easy": 86.59605824557663,
       "hard": 62.546614906669795,
       "judge": 68.82118654512506,
       "standard": 68.17920024120974
      }
     }
    }
   },
   "jevbench_score": 76.44111138489434,
   "key": "fastino-glide-api",
   "last_measured_on": "2026-10-22",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.40",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://fastino.ai#fastino-glide-api",
   "noul_decisive": {
    "acc_among_decisive": 0.9749216300940439,
    "decisive": 319,
    "decisive_rate": 0.8621621621621621,
    "n": 373,
    "supported": true,
    "valid": 370
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 2,
    "axes": {
     "calibration": 93.01106628088438,
     "cost": 59.111125613009825,
     "intelligence": 76.39198423827783,
     "speed": 87.30503336667806
    },
    "board_release": "v1.6.43",
    "capability": 84.7015252595811,
    "capability_api_board_rank": 2,
    "capability_open_board_rank": null,
    "capability_rank": 4,
    "composite_ci95": {
     "A": [
      75.0560545503276,
      77.30340620880787
     ],
     "B": [
      74.53280727316283,
      77.76808823284206
     ],
     "C": [
      75.0560545503276,
      77.30340620880787
     ]
    },
    "jevbench_score": 76.60902426459442,
    "measured_in": "v1.6.40",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 2,
    "ranks": {
     "A": 2,
     "B": 2,
     "C": 2,
     "capability": 9
    },
    "scores": {
     "A": 76.60902426459442,
     "B": 76.56551765292035,
     "C": 76.60902426459442
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.3601445150561631,
     "p50_s_raw": 0.3601445150561631,
     "p95_s_adjusted": 0.5164399832952765,
     "p95_s_raw": 0.5164399832952765
    },
    "views": {
     "Balanced 33:33:33": 72.35582674096251,
     "Emphasis on Accuracy 60:20:20": 73.91800482728986,
     "Emphasis on Cost 20:20:60": 66.40429071449509,
     "Emphasis on Speed 20:60:20": 77.67599908573227,
     "Intelligence only": 76.39198423827783
    }
   },
   "rank": 3,
   "ranked": true,
   "ranks": {
    "A": 3,
    "B": 3,
    "C": 3,
    "capability": 11
   },
   "repo": "https://fastino.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 76.44111138489434,
    "B": 76.36628298199761,
    "C": 76.44111138489434
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1497,
    "p50_s_adjusted": 0.35695503698661923,
    "p50_s_raw": 0.35695503698661923,
    "p95_s_adjusted": 0.5584349319804464,
    "p95_s_raw": 0.5584349319804464
   },
   "status": {
    "answered_ok": 1480,
    "missing": 0,
    "rows": 1497,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API (POST api.fastino.ai/v1/systemone, model fastino/GLiDE) through the unchanged TypeSafe adapter with one added request field, store:false; no vendor code ran locally. SCORED ON 1,497 OF THE DRAW'S 1,500 ITEMS: 3 items Fastino had already seen through its hosted GLiNER-2.5-Decide on 8 Oct were never sent (custody rule: an item a provider has seen is never scored for that provider). The row is scored with the unmodified frozen scorer on the draw's gold without those 3 items (a second invocation of the same scorer; the other rows of this board are scored on all 1,500 items). PROVIDER-HOSTED: Fastino's API serves the model as `glide_v2` (every answered response reported it); the weights and serving were not inspected, and this is not the open-weights GLiDE 1.2 release, which is a separate pending measurement. RETENTION: Fastino's default is to retain API inputs; store:false on every request; the vendor's not-stored response header was enforced on every answered request by the run harness (error responses carry no such header by the vendor's documentation). NOT ANSWERED, counted wrong: 15 items exceed the API's 65,536-token prompt limit (HTTP 400 'prompt exceeds the maximum context length'), counted wrong; and 2 gateway errors (HTTP 502) from Fastino's API, not retried (one pass), counted wrong. PRICE CHANGED SINCE THE FIRST MEASUREMENT: on 6 Oct 2026 this vendor's GLiDE was listed at USD 0.15 per 1M input tokens with a 40,000-token prompt limit and scored on 600 items (key fastino-glide, Capability 77.0, outside the cost cap); today's tariff is USD 0.04 per 1M input tokens with a 65,536-token limit, and this row is a new measurement on the full draw. The older row stays as history. Latency was measured from Sandy (Germany) with no API adjustment, while the evaluation host was busy with other work; 2 answered decisions took more than 10 seconds (21.15 s and 76.42 s); v1.6 applies no 10-second rule. Native probabilities on all three question types, no generated tokens; the customer-side statements about the model are the vendor's and are not independently verified. SEALED ITEMS SENT TO THE VENDOR: the API sees the 1,197 sealed items it was sent; acceptable because every sealed item of this draw was already retired from scoring.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": -5.961080915580955,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": -209.54990941340955
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 61.30216821922519,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 43.99999999999999,
        "score": 65.35719394717714
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.30216821922519
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 85.67892146745031,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 85.18518518518519,
        "score": 75.10361173749092
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.67892146745031
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.29471891787144,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 98.88415675361429
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.29471891787144
      },
      "jaggedness": {
       "accuracy": 0.7944444444444444,
       "cc": 58.69250954076001,
       "cc_by_type": {
        "choice": 79.78825794032724,
        "noul": 18.918918918918926,
        "score": 77.37035176303387
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.69250954076001
      },
      "llm_router": {
       "accuracy": 0.952076677316294,
       "cc": 88.04795205712485,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 100.0,
        "score": 67.31370736447127
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 88.04795205712485
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7916666666666666,
       "cc": 52.713148360991475,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 33.33333333333333,
        "score": 49.548379790878194
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 6,
        "score": 7
       },
       "score": 52.713148360991475
      },
      "multilingual": {
       "accuracy": 0.8218181818181818,
       "cc": 69.62686351470224,
       "cc_by_type": {
        "choice": 85.32768177372026,
        "noul": 55.55555555555556,
        "score": 67.99735321483094
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.62686351470224
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 74.54014406808,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 73.33333333333334,
        "score": 50.28709887090663
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.54014406808
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 81.53200714105311,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 80.0,
        "score": 79.6110364381744
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.53200714105311
      },
      "usecase": {
       "accuracy": 0.8636363636363636,
       "cc": 74.04969523790491,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 58.33333333333333,
        "score": 73.61967394900884
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 74.04969523790491
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 49.28337524952835,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": 16.897744796204094
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 49.28337524952835
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 55.50357736254705,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 33.17739875430783
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 55.50357736254705
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 93.40496888083449,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100.0,
        "score": 88.17950841241498
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 93.40496888083449
      },
      "el": {
       "accuracy": 0.9333333333333333,
       "cc": 94.3834224733954,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100.0,
        "score": 99.91004395649901
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 94.3834224733954
      },
      "en": {
       "accuracy": 0.8740225890529973,
       "cc": 76.08356533618894,
       "cc_by_type": {
        "choice": 88.57714697160152,
        "noul": 68.34532374100719,
        "score": 71.3282252959581
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 76.08356533618894
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 74.21236753974422,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 32.425030841092365
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 74.21236753974422
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.72593685668032,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.84447723670762
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.72593685668032
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 92.06735412018907,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 97.37853294880247
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 92.06735412018907
      },
      "hi": {
       "accuracy": 0.8888888888888888,
       "cc": 70.34650149976363,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 66.66666666666667,
        "score": 44.37283783262424
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 4
       },
       "score": 70.34650149976363
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 66.41366853069502,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 13.27062367470393
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 66.41366853069502
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 47.24100841291983,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 19.999999999999996,
        "score": 36.020086398410015
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 47.24100841291983
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 18.987371364068355,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 56.96211409220508
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.987371364068355
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.96955130238082,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.9086539071425
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.96955130238082
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 1.7442176532610603,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -35.45998773285752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.7442176532610603
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 78.64726523260191,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 71.17666662459372
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 78.64726523260191
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 78.0409563759746,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 65.9437294697034
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 78.0409563759746
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 59.78378036796139,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 44.8832559975012
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 59.78378036796139
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 85.72572454543065,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 78.91630407107458
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 85.72572454543065
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.93118510075651,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 84.05161981839854
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.93118510075651
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 65.20564880948768,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 62.28361309512973
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 65.20564880948768
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.9066666666666666,
       "cc": 87.9446602499091,
       "cc_by_type": {
        "choice": 87.9446602499091
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 87.9446602499091
      },
      "noul": {
       "accuracy": 0.8337801608579088,
       "cc": 66.75603217158177,
       "cc_by_type": {
        "noul": 66.75603217158177
       },
       "low_n": false,
       "n": 373,
       "n_by_type": {
        "noul": 373
       },
       "score": 66.75603217158177
      },
      "score": {
       "accuracy": 0.8074866310160428,
       "cc": 69.47514714474733,
       "cc_by_type": {
        "score": 69.47514714474733
       },
       "low_n": false,
       "n": 374,
       "n_by_type": {
        "score": 374
       },
       "score": 69.47514714474733
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 28.27943132511149,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -25.0,
        "score": 41.83829397533446
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 28.27943132511149
      },
      "code_linting": {
       "accuracy": 1.0,
       "cc": 93.28648102165937,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 79.8594430649781
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 93.28648102165937
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 64.14435699593669,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 24.953396191062115
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 64.14435699593669
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -4.4326718070937305,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": -88.15181659087183
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.875,
       "cc": 68.1764684457371,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 85.68882562706641
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 68.1764684457371
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 56.88352721922263,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 49.89586467653584
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 56.88352721922263
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 51.09999092184478,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -14.28571428571429,
        "score": 67.58568705124861
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.09999092184478
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 52.69076696339014,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 58.07230089017044
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 52.69076696339014
      },
      "insurance_claims": {
       "accuracy": 0.6428571428571429,
       "cc": 62.65779285636237,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 50.0,
        "score": 58.80671190242044
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 4
       },
       "score": 62.65779285636237
      },
      "knowledge_graphs": {
       "accuracy": 0.9545454545454546,
       "cc": 90.65854177937764,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 71.97562533813296
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 90.65854177937764
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.60857584215633,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 58.89807894765766
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.60857584215633
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 45.5354720458297,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 67.78921183641384
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 45.5354720458297
      },
      "llm_guardrails": {
       "accuracy": 0.9629629629629629,
       "cc": 80.95439830812018,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 82.8631949243605
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 80.95439830812018
      },
      "model_routing": {
       "accuracy": 0.9351851851851852,
       "cc": 84.8410274166256,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 100.0,
        "score": 58.02957575637029
       },
       "low_n": false,
       "n": 216,
       "n_by_type": {
        "choice": 101,
        "noul": 65,
        "score": 50
       },
       "score": 84.8410274166256
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 55.15593532195087,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 88.50043552439388
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 55.15593532195087
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 67.57303995857968,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 69.38578654240574
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 67.57303995857968
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 62.24567525530753,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 36.7370257659226
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 62.24567525530753
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 56.083255830130874,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 63.901941403436105
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 56.083255830130874
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.92021847495123,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.76065542485364
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.92021847495123
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.84523935751662,
      93.09512159193915
     ],
     "I": [
      71.97632052544036,
      79.51320538269066
     ],
     "capability": [
      81.08718032572618,
      85.69953649578142
     ],
     "composite": {
      "A": [
       74.90033635084451,
       77.08874894746072
      ],
      "B": [
       74.3293214235386,
       77.56901714437397
      ],
      "C": [
       74.90033635084451,
       77.08874894746072
      ]
     },
     "seed": 16
    },
    "ci95_n": 1197,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1497,
    "n_public": 300,
    "n_sealed_side": 1197,
    "on_S": {
     "calibration": 92.84353248115252,
     "capability": 84.45598056597845,
     "intelligence": 76.06842865080436
    },
    "per_type": {
     "choice": {
      "accuracy": 0.9066666666666666,
      "calibration": 91.30791369573681,
      "cc": 87.9446602499091,
      "cc_by_type": {
       "choice": 87.9446602499091
      },
      "cc_tiered": 88.71900306704876,
      "cc_tiered_open": 90.51495356571994,
      "cc_tiered_sealed": 86.92305256837759,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 87.9446602499091
     },
     "noul": {
      "accuracy": 0.8337801608579088,
      "calibration": 92.12139262649261,
      "cc": 66.75603217158177,
      "cc_by_type": {
       "noul": 66.75603217158177
      },
      "cc_tiered": 69.0624696096989,
      "cc_tiered_open": 75.94824016563147,
      "cc_tiered_sealed": 62.1766990537663,
      "low_n": false,
      "n": 373,
      "n_by_type": {
       "noul": 373
      },
      "score": 66.75603217158177
     },
     "score": {
      "accuracy": 0.8074866310160428,
      "calibration": 95.10129112122816,
      "cc": 69.47514714474733,
      "cc_by_type": {
       "score": 69.47514714474733
      },
      "cc_tiered": 70.42381327566545,
      "cc_tiered_open": 72.88717875232587,
      "cc_tiered_sealed": 67.96044779900504,
      "low_n": false,
      "n": 374,
      "n_by_type": {
       "score": 374
      },
      "score": 69.47514714474733
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.016000000000000014,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.008042895442359255,
     "n": 373
    },
    "score": {
     "invalid_rate": 0.010695187165775444,
     "n": 374
    }
   },
   "views": {
    "Balanced 33:33:33": 72.18991121677152,
    "Emphasis on Accuracy 60:20:20": 73.69286891912549,
    "Emphasis on Cost 20:20:60": 66.32035800171175,
    "Emphasis on Speed 20:60:20": 77.46597567840699,
    "Intelligence only": 76.06842865080436
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Goddragon (Hugging Face)",
   "axes": {
    "calibration": 77.19519806916665,
    "cost": 66.19900073922123,
    "intelligence": 27.391250053452495,
    "speed": 92.05594930781893
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12453810152460322,
      "ece_score": 75.09237969507936,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 75.09237969507936,
      "score_before_time_limit": 75.09237969507936,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3512064343163539,
      "brier": 0.1726437405208735,
      "ece": 0.11296355354302531,
      "n": 373,
      "score": 77.40728929139493,
      "score_before_time_limit": 77.40728929139493,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8790322580645161,
      "interval90_width": 0.48521505376344093,
      "n": 372,
      "nrps": 0.12849056705545897,
      "score": 79.08592522102566,
      "score_before_time_limit": 79.08592522102566,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1448954642620138
     }
    },
    "score": 77.19519806916665
   },
   "capability": 52.29322406130957,
   "composite_ci95": {
    "A": [
     9.860481987953724,
     22.96523291949192
    ],
    "B": [
     8.068319045554896,
     19.830397611972373
    ],
    "C": [
     6.847556936078973,
     15.94807841631383
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 1f4d1f7b1ce97ab27b938043444b9c2757e4a9dbecc799a803122588401f5bf1, 20 Oct additive re-freeze; this base resolves identically under the earlier 6202ef78...): base string Qwen/Qwen3.5-4B (the authors' stated Qwen3.5-4B / NeoHorse lineage, unverified) priced as Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author server's own usage block: 662,682 input on the 1,485 answered items (equal to our offline count through the author's encode() item by item, pre-registered), 0 output tokens (one forward pass with a pointer readout, no generation); the 15 refused items count 0 tokens. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them in full), which this author's server refuses; their tokens count for no row's cost.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.013253640000000002
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013387515151515154
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Arbiter-4B (Goddragon, NeoHorse 4B pointer model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4 80 GB (FIN-02, Finland, EU, pay-as-you-go non-spot, USD 2.168/h), shared by the two NeoHorse-family rows as two separate back-to-back uses (sealed input deleted from the node between them). The author's own server.py as shipped on stock PyPI vLLM 0.30.0 / torch 2.13.0 in a pre-built image, container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only bundle, all weight shards sha256-verified; driven by the unchanged pinned evaluator driver, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.1215764916093,
    "I_sealed": 22.660923615295694,
    "base": 27.391250053452495,
    "excess": -0.37887389404210836,
    "gap": 9.460652876313603,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 58.95334556281439,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 47.15319282793332,
       "judge": 56.49307844429796,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -21.739130434782606,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 37.41138391201351,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.07472571843502,
       "hard": 42.643741131070314,
       "judge": 32.30337526144111,
       "standard": 24.277011546547755
      }
     },
     "sealed|choice": {
      "cc": 46.87143166012293,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 69.50444726810673,
       "hard": 40.67306580316872,
       "judge": 48.87365328109698,
       "standard": 44.94832313857837
      }
     },
     "sealed|noul": {
      "cc": 4.770267303463389,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 1.8181818181818077,
       "judge": -16.92307692307692,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 16.341071882300767,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 34.31316593972345,
       "hard": 7.707251567970719,
       "judge": 10.049632979159162,
       "standard": 34.059823836961925
      }
     }
    }
   },
   "jevbench_score": 15.914507055188015,
   "key": "goddragon-arbiter-4b",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (HF metadata and LICENSE), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Goddragon/Arbiter-4B@8fd20568f359f4156de55f016027efdc841d690a",
   "noul_decisive": {
    "acc_among_decisive": 0.8429752066115702,
    "decisive": 242,
    "decisive_rate": 0.6487935656836461,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.19519806916665,
     "cost": 66.19900073922123,
     "intelligence": 27.391250053452495,
     "speed": 92.16491010646149
    },
    "board_release": "v1.6.43",
    "capability": 52.29322406130957,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 89,
    "capability_rank": 115,
    "composite_ci95": {
     "A": [
      9.862003577191583,
      22.969397756248135
     ],
     "B": [
      8.069131709312094,
      19.832903146488743
     ],
     "C": [
      6.848613595271931,
      15.950970664061202
     ]
    },
    "jevbench_score": 15.917217056211483,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": 117,
    "rank": 155,
    "ranks": {
     "A": 155,
     "B": 158,
     "C": 155,
     "capability": 170
    },
    "scores": {
     "A": 15.917217056211483,
     "B": 13.406692451289457,
     "C": 11.053622955702421
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24398728700034553,
     "p50_s_raw": 0.046993643500172766,
     "p95_s_adjusted": 0.24896700299996155,
     "p95_s_raw": 0.04948350149998077
    },
    "views": {
     "Balanced 33:33:33": 14.413667848999275,
     "Emphasis on Accuracy 60:20:20": 11.075877822934517,
     "Emphasis on Cost 20:20:60": 16.19147002227193,
     "Emphasis on Speed 20:60:20": 17.828945195366323,
     "Intelligence only": 8.22044918523227
    }
   },
   "rank": 173,
   "ranked": true,
   "ranks": {
    "A": 173,
    "B": 175,
    "C": 174,
    "capability": 190
   },
   "repo": "https://huggingface.co/Goddragon/Arbiter-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.914507055188015,
    "B": 13.405154322939076,
    "C": 11.051741010547234
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24426879499983442,
    "p50_s_raw": 0.04713439749991721,
    "p95_s_adjusted": 0.2549981790999709,
    "p95_s_raw": 0.05249908954998545
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server vllm-fast/server.py at 8fd20568f359f4156de55f016027efdc841d690a, started directly as shipped (independent static review PASS WITH FINDINGS, 0 blockers; no subprocess/network in the server import path; run_evaluation.py, evaluate.py and provenance/* never started). Author defaults kept: position pooling ON (its shipped default), no prefix or head cache. BASE: the repository's backbone/config.json modification notice states that the language-model weights were fine-tuned from Qwen/Qwen3.5-4B (tensor values unchanged by repackaging), and the card's base_model field names the NeoHorse lineage (TokenRhythm/NeoHorse-Jev-4B); the base string Qwen/Qwen3.5-4B is used because it is the upstream model the repository itself states and the only one the frozen pricing reference can resolve. Lineage claims are the authors' and unverified (the manifest names only a local path). FIRST MEASUREMENT (free-queue submission 574B4820). CONTEXT CAP: the author server refuses requests whose state exceeds 8,192 tokens or whose state+question row exceeds 12,288 tokens (HTTP 422); on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like d3-turbo, elf-1-12b and seb-9b on this draw. SAME FAMILY: Goddragon/Arbiter-4B and roudone/Decider-4B are two different checkpoints submitted through the free queue that both serve as 'NeoHorse-Jev-4B' (TYPESAFE_MODEL alias set for the frozen adapter). PROVENANCE, author statements (unverified): base lineage TokenRhythm/NeoHorse-Jev-4B / Qwen3.5-4B; training on the typed-decisions train split plus classification sets, TRAM, HelpSteer2 and constructed examples (README). No rule against public items was found in the reviewed code. HARDWARE: A100 80 GB (the RTX PRO 6000 was out of stock); latency reflects that card. Shipped temperature 1.1942 is the author's choice (their own held-out ECE got worse with it).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5454545454545454,
       "cc": 13.1573417948352,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": 100.0,
        "score": -93.86130794882774
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 13.1573417948352
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 6.759864864914857,
       "cc_by_type": {
        "choice": 30.010604453870634,
        "noul": -12.0,
        "score": 2.268990140873939
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 6.759864864914857
      },
      "composite_gen": {
       "accuracy": 0.40594059405940597,
       "cc": -11.562426232861531,
       "cc_by_type": {
        "choice": 54.47154471544716,
        "noul": -48.14814814814815,
        "score": -41.0106752658836
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.6875,
       "cc": 42.72141227498651,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -19.999999999999996,
        "score": 48.16423682495953
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 42.72141227498651
      },
      "jaggedness": {
       "accuracy": 0.37777777777777777,
       "cc": 6.9964087476416905,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -13.513513513513509,
        "score": 19.68079557934521
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 6.9964087476416905
      },
      "llm_router": {
       "accuracy": 0.7635782747603834,
       "cc": 65.55835613406727,
       "cc_by_type": {
        "choice": 61.169322884564586,
        "noul": 79.38144329896907,
        "score": 56.124302218668156
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 65.55835613406727
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -30.50844844645876,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -50.0,
        "score": -42.55627317442783
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5818181818181818,
       "cc": 25.353468409646783,
       "cc_by_type": {
        "choice": 58.9175089664167,
        "noul": -19.444444444444443,
        "score": 36.58734070696811
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 25.353468409646783
      },
      "negation_pair_gen": {
       "accuracy": 0.4883720930232558,
       "cc": -8.074110162754854,
       "cc_by_type": {
        "choice": 53.333333333333336,
        "noul": -33.333333333333336,
        "score": -44.22233048826456
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.625,
       "cc": 52.06391478029295,
       "cc_by_type": {
        "choice": 52.06391478029295
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 52.06391478029295
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 65.2669271494847,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 60.00000000000001,
        "score": 65.83081147848414
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 65.2669271494847
      },
      "usecase": {
       "accuracy": 0.5181818181818182,
       "cc": 26.957730500631396,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": 0.0,
        "score": 29.892799345031406
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 26.957730500631396
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -38.476656738814235,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -50.0,
        "score": -70.19187497834747
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 56.66171289152569,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 36.65180534124374
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 56.66171289152569
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 65.80687892506546,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 75.0,
        "score": 70.20824739466542
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 65.80687892506546
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 23.999811413863814,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 88.94543051719666
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 23.999811413863814
      },
      "en": {
       "accuracy": 0.5490877497827976,
       "cc": 26.586689973573968,
       "cc_by_type": {
        "choice": 47.059469618383936,
        "noul": 12.230215827338121,
        "score": 20.470384474999847
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 26.586689973573968
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 58.68217680179415,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": -4.377613150898152
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 58.68217680179415
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 14.608201601683332,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 9.944823384284973
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 14.608201601683332
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 45.62934066592817,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 58.06449258601979
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 45.62934066592817
      },
      "hi": {
       "accuracy": 0.38095238095238093,
       "cc": -23.312194873205982,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50.0,
        "score": -69.65725501067939
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 75.85729665962164,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 55.63112614410264
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 75.85729665962164
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 20.6313226795196,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -19.999999999999996,
        "score": 53.37927383681138
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 20.6313226795196
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -8.161926992907732,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 19.95866346572126
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.54315187844138,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 84.29612230199078
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.54315187844138
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -6.316997789459312,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 7.0230326056480274
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 20.232627552011937,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40.0,
        "score": 31.16762450961178
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 20.232627552011937
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 49.75002180092513,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 31.677879191814885
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 49.75002180092513
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 2.1082100916838846,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 28.45228984951973
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 2.1082100916838846
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 29.533493664601565,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50.0,
        "score": -17.92125813663008
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.533493664601565
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 62.240877851683614,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 51.238762587308884
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.240877851683614
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 19.184165055251437,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": -24.93338054046037
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.184165055251437
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.608,
       "cc": 49.36757304961825,
       "cc_by_type": {
        "choice": 49.36757304961825
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 49.36757304961825
      },
      "noul": {
       "accuracy": 0.544,
       "cc": 8.800000000000008,
       "cc_by_type": {
        "noul": 8.800000000000008
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 8.800000000000008
      },
      "score": {
       "accuracy": 0.472,
       "cc": 22.702016645247348,
       "cc_by_type": {
        "score": 22.702016645247348
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 22.702016645247348
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -0.8162936667601267,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -50.0,
        "score": -4.44888100028038
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 1.8108502810784488,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": 11.111111111111116,
        "score": -12.51085840452173
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.8108502810784488
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 45.134859392293265,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 0.0,
        "score": 51.6647407785058
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 45.134859392293265
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -38.24161193978049,
       "cc_by_type": {
        "choice": -28.65497076023392,
        "noul": -100.0,
        "score": 13.930134940892447
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 19.891098676055922,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 21.9921366078779
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 19.891098676055922
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": 16.539907187567064,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -66.66666666666667,
        "score": 54.02223728597164
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 16.539907187567064
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 18.7768282505638,
       "cc_by_type": {
        "choice": 41.17647058823529,
        "noul": -14.28571428571429,
        "score": 29.4397284491704
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 18.7768282505638
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -19.139393677670046,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100.0,
        "score": 26.2055123467808
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 21.22332456763144,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 46.16997370289432
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 21.22332456763144
      },
      "knowledge_graphs": {
       "accuracy": 0.2727272727272727,
       "cc": 9.518799890413932,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 33.33333333333333,
        "score": -1.9932077520272928
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 9.518799890413932
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 20.02190004982577,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 5.026940459554852
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 20.02190004982577
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -25.972993526020257,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -47.811453696340344
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 28.56631129583379,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 40.24787822531327
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 28.56631129583379
      },
      "model_routing": {
       "accuracy": 0.7926267281105991,
       "cc": 66.41479171839013,
       "cc_by_type": {
        "choice": 74.28571428571429,
        "noul": 84.84848484848484,
        "score": 40.11017602097127
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.41479171839013
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 4.953745673457386,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": 72.44281091672534
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 4.953745673457386
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 49.47362762453108,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 48.513733384271085
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 49.47362762453108
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 3.6459962938239614,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50.0,
        "score": -24.283685995375404
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 3.6459962938239614
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 12.90172875431424,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 15.516780465841274
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 12.90172875431424
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 77.49722676858175,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 82.49168030574525
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.49722676858175
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.25547879639667,
      80.42268223181145
     ],
     "I": [
      22.680931701137304,
      31.747668356672595
     ],
     "capability": [
      47.99634931098826,
      55.30745997453868
     ],
     "composite": {
      "A": [
       9.860481987953724,
       22.96523291949192
      ],
      "B": [
       8.068319045554896,
       19.830397611972373
      ],
      "C": [
       6.847556936078973,
       15.94807841631383
      ]
     },
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.19519806916665,
     "capability": 52.29322406130957,
     "intelligence": 27.391250053452495
    },
    "per_type": {
     "choice": {
      "accuracy": 0.608,
      "calibration": 75.09237969507936,
      "cc": 49.36757304961825,
      "cc_by_type": {
       "choice": 49.36757304961825
      },
      "cc_tiered": 52.91238861146866,
      "cc_tiered_open": 58.95334556281439,
      "cc_tiered_sealed": 46.87143166012293,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.36757304961825
     },
     "noul": {
      "accuracy": 0.544,
      "calibration": 77.40728929139493,
      "cc": 8.800000000000008,
      "cc_by_type": {
       "noul": 8.800000000000008
      },
      "cc_tiered": 2.3851336517316946,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 4.770267303463389,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 8.800000000000008
     },
     "score": {
      "accuracy": 0.472,
      "calibration": 79.08592522102566,
      "cc": 22.702016645247348,
      "cc_by_type": {
       "score": 22.702016645247348
      },
      "cc_tiered": 26.87622789715714,
      "cc_tiered_open": 37.41138391201351,
      "cc_tiered_sealed": 16.341071882300767,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.702016645247348
     }
    },
    "raw_on_A": {
     "calibration": 74.72248487244359,
     "capability": 51.694359455861786,
     "intelligence": 28.666234039279978
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.410705013087656,
    "Emphasis on Accuracy 60:20:20": 11.07482800434468,
    "Emphasis on Cost 20:20:60": 16.18922659758732,
    "Emphasis on Speed 20:60:20": 17.820787398141796,
    "Intelligence only": 8.22044918523227
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "H2O.ai (paid fast-lane order 5fa1a5f0, PUBLIC visibility)",
   "axes": {
    "calibration": 89.37957627728649,
    "cost": 50.57266115000792,
    "intelligence": 79.29803124474094,
    "speed": 89.5083860213906
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0321064307460744,
      "ece_score": 93.57871385078512,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.33099766610799,
      "score_before_time_limit": 93.57871385078512,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.06538327479433943,
      "ece": 0.05774715480204895,
      "n": 375,
      "score": 87.97883267137907,
      "score_before_time_limit": 88.4505690395902,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.96,
      "interval90_width": 0.15333333333333338,
      "n": 375,
      "nrps": 0.04444906362089437,
      "score": 87.82889849437241,
      "score_before_time_limit": 88.53719606287541,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.0924035075607986
     }
    },
    "score": 89.37957627728649
   },
   "capability": 84.33880376101371,
   "composite_ci95": {
    "A": [
     71.77822793115342,
     73.87651780604975
    ],
    "B": [
     72.53976200330692,
     75.43964488054165
    ],
    "C": [
     71.77822793115342,
     73.87651780604975
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7): declared base google/gemma-4-31B-it -> reference() -> google/gemma-4-31b-it, USD 0.09 per million input tokens, USD 0.34 per million output tokens (frozen price cut-off snapshot), the same reference as the other Gemma-4-31B-it rows of this board. The customer asked for the cost of the base model. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author's server usage block (the full record, 1 output token per decision for the readout). Over all 1,500 items the frozen resolve() gives USD 0.115415 per 1,000 decisions; the official cost axis uses the common 1,485-item basis (below), where the row is under the official cost cap. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 727,319 input / 1,485 output tokens. All-item usage 1,917,924 input / 1,500 output over 1,500 items; the 15 longest items, which this row answered in full, count for no row's cost.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.11541544000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.044419939393939394
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "H2O-Lightning-31B v1.0.7 (H2O.ai, Gemma-4-31B-it fine-tune, vLLM + h2oai overlay, one-engine FP8 profile, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane order 5fa1a5f0 (H2O.ai). Evaluator-owned rented Verda GPU VM (1RTXPRO6000.30V, FIN-03, Finland, EU, pay-as-you-go, USD 2.535/h), 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB). Run image built on the digest-pinned official vllm/vllm-openai:v0.30.0 with the author's overlay and shim installed before any test item existed, all 26 weight files sha256-verified at revision ca6e6f6e, then run in an offline container (network none), the cut proven from inside before the sealed input and again before the scored run. Driven serially, one request at a time, by the unchanged self-hosted runner. Measured 22 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; raw latency p50 0.065 s, p95 0.126 s, maximum 39.08 s. The standard self-hosted x2 + 0.15 s adjustment applies (assumption, not measured).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.9765674204135,
    "I_sealed": 78.61949506906838,
    "base": 79.29803124474094,
    "excess": -8.482454419010587,
    "gap": 1.3570723513451242,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 85.05319626115181,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 79.2387543252595,
       "judge": 84.17930125247199,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 85.29951690821257,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 70.0,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 69.57698909187614,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.94393070994124,
       "hard": 80.8292257979457,
       "judge": 68.24978094093717,
       "standard": 52.37985709711295
      }
     },
     "sealed|choice": {
      "cc": 86.12500463167756,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 82.82641378512778,
       "judge": 86.77766895200784,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 81.43542715647708,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 74.54545454545453,
       "judge": 84.61538461538463,
       "standard": 88.05970149253733
      }
     },
     "sealed|score": {
      "cc": 68.2980534190505,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 86.2967685525146,
       "hard": 66.72128429528557,
       "judge": 62.76666308088821,
       "standard": 70.74931960709175
      }
     }
    }
   },
   "jevbench_score": 73.06662231286829,
   "key": "h2o-lightning-31b",
   "last_measured_on": "2026-10-22",
   "licence": "H2O Model Non-Commercial Licence v1.1 (weights; gated on Hugging Face; benchmarking and publishing results are allowed by the licence). Base Gemma 4 under its own licence.",
   "listing": "ranked",
   "measured_in": "v1.6.39",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "h2oai/h2o-lightning-31b@ca6e6f6e8e08e2d21c8c9f45261e05444626647a",
   "noul_decisive": {
    "acc_among_decisive": 0.9195710455764075,
    "decisive": 373,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.18882631775024,
     "cost": 50.57266115000792,
     "intelligence": 80.15012041049178,
     "speed": 89.8424830803211
    },
    "board_release": "v1.6.43",
    "capability": 85.16947336412102,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 2,
    "capability_rank": 3,
    "composite_ci95": {
     "A": [
      72.2022562910307,
      74.22532314173431
     ],
     "B": [
      73.01307775885572,
      75.86853392037305
     ],
     "C": [
      72.2022562910307,
      74.22532314173431
     ]
    },
    "jevbench_score": 73.43686411111796,
    "measured_in": "v1.6.39",
    "method": "v1.6",
    "open_board_rank": 3,
    "rank": 12,
    "ranks": {
     "A": 12,
     "B": 6,
     "C": 12,
     "capability": 8
    },
    "scores": {
     "A": 73.43686411111796,
     "B": 74.68801579994582,
     "C": 73.43686411111796
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27608948999968563,
     "p50_s_raw": 0.0630447449998428,
     "p95_s_adjusted": 0.37557944640020646,
     "p95_s_raw": 0.11278972320010325
    },
    "views": {
     "Balanced 33:33:33": 69.15516340580727,
     "Emphasis on Accuracy 60:20:20": 73.17014211031464,
     "Emphasis on Cost 20:20:60": 60.293435536349946,
     "Emphasis on Speed 20:60:20": 76.1708694366029,
     "Intelligence only": 80.15012041049178
    }
   },
   "rank": 14,
   "ranked": true,
   "ranks": {
    "A": 14,
    "B": 8,
    "C": 14,
    "capability": 12
   },
   "repo": "https://huggingface.co/h2oai/h2o-lightning-31b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.06662231286829,
    "B": 74.23330468978295,
    "C": 73.06662231286829
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27901971299970685,
    "p50_s_raw": 0.06450985649985341,
    "p95_s_adjusted": 0.40135299689954984,
    "p95_s_raw": 0.12567649844977494
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's documented JevBench configuration, unmodified: run_default.sh, the one-engine profile (author's README, 'how to run for JevBench'), on vLLM 0.30.0 (official vllm/vllm-openai:v0.30.0 image, digest-pinned) with the author's h2oai/vllm overlay at commit 6bc6d5a8 (18 patched files, sha256-checked against the pin) and the author's shim lightning_31b_shim.py; torch 2.13.0, transformers 5.20.0, torchcodec 0.17.0 hash-locked. Engine: one vLLM engine, FP8 weights (vLLM online FP8 of the bf16 checkpoint, --quantization fp8), two-way reading by default (one-way, read whole, for a record over the 8,192-token two-way window or a request over the 4,096-token latency cap); the profile's own temperatures and yes/no floor 0.801 as shipped. Static review of every author file found no network, subprocess, eval, pickle or trust_remote_code use and no branch on item text. MEASURED AS SHIPPED: the author's own documented JevBench configuration (run_default.sh, one-engine profile): FP8 weights and the two-way reading that this profile applies by default. A different serving choice (for example the author's stock-vLLM one-way fallback) would give different numbers. ALL 1,500 ITEMS ANSWERED IN FULL: none was refused, cut or truncated; the 15 longest items (76,497 to 81,692 input tokens, below the engine's 126,976-token cap) were read whole one-way. 15 decisions took more than 10 seconds (the 15 longest items, maximum 39.08 s); v1.6 applies no 10-second rule. PUBLIC VS SEALED, author statement: the author's README reports 214 of 231 on the public JevBench items from its own client (easy 48/48, original 72/72, hard 94/111). That figure is the author's, on the public items; this row's numbers are what we measured on the sealed draw, and the scorer's standard public-vs-sealed gap rule applies as for every row. TRAINING DATA: the author's training-data statement was not checked. Customer statements about the model, its training and its serving are not independently verified. Measured accuracy on the 15 longest items (answered in full): 7 of 15 correct (46.7 %), versus 1313 of 1485 (88.4 %) on the remaining items; a Score item is correct within half a label step.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 69.31035450645739,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 7.931063519372161
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 69.31035450645739
      },
      "calibration_gen": {
       "accuracy": 0.9702970297029703,
       "cc": 92.23797710576798,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92.0,
        "score": 87.89526747849166
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 92.23797710576798
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 89.44016777438448,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 100.0,
        "score": 74.82456836380385
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.44016777438448
      },
      "fairness": {
       "accuracy": 0.9791666666666666,
       "cc": 94.31870784276252,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 86.66666666666667,
        "score": 96.28945686162085
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 94.31870784276252
      },
      "jaggedness": {
       "accuracy": 0.8277777777777777,
       "cc": 72.03555111247822,
       "cc_by_type": {
        "choice": 76.90086621751685,
        "noul": 67.56756756756756,
        "score": 71.63821955235024
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 72.03555111247822
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 82.4432474217307,
       "cc_by_type": {
        "choice": 89.69798362243552,
        "noul": 97.93814432989691,
        "score": 59.69361431285969
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 82.4432474217307
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 57.01060158493846,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": 100.0,
        "score": 32.887474857908195
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 57.01060158493846
      },
      "multilingual": {
       "accuracy": 0.8472727272727273,
       "cc": 74.4939896452101,
       "cc_by_type": {
        "choice": 84.3495272253016,
        "noul": 69.44444444444444,
        "score": 69.68799726588428
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 74.4939896452101
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 79.72153832282477,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 73.33333333333334,
        "score": 65.83128163514097
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 79.72153832282477
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.953125,
       "cc": 92.80164393999273,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100.0,
        "score": 84.41093782598415
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 92.80164393999273
      },
      "usecase": {
       "accuracy": 0.8227272727272728,
       "cc": 75.452467331574,
       "cc_by_type": {
        "choice": 80.39215686274511,
        "noul": 75.0,
        "score": 70.96524513197689
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 75.452467331574
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 27.695577686653905,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -9.770409797181134
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.695577686653905
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 86.6601798445292,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 80.7178667225277
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 86.6601798445292
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 90.54110541685141,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100.0,
        "score": 79.58791802046574
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.54110541685141
      },
      "el": {
       "accuracy": 0.8,
       "cc": 76.74015887855013,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 97.07336304160941
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 76.74015887855013
      },
      "en": {
       "accuracy": 0.8853171155516942,
       "cc": 81.4003355140322,
       "cc_by_type": {
        "choice": 87.25912546832475,
        "noul": 84.89208633093526,
        "score": 72.0497947428366
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 81.4003355140322
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 65.82902920546668,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 7.27501583825978
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 65.82902920546668
      },
      "fi": {
       "accuracy": 1.0,
       "cc": 99.04348580348609,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.1304574104583
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 99.04348580348609
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.01638557859114,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 94.40209791224403
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.01638557859114
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 74.74309771694028,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 100.0,
        "score": 57.74884622344651
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 74.74309771694028
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 78.05757894622408,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 34.17273683867226
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 78.05757894622408
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 65.43997328394065,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 80.0,
        "score": 16.31991985182193
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 65.43997328394065
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 61.36020671382338,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 84.08062014147015
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 61.36020671382338
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 98.99576194094993,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.98728582284978
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.99576194094993
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 36.274143337969065,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 42.15576334724053
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 36.274143337969065
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8974358974358975,
       "cc": 83.23634974536745,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80.0,
        "score": 77.32648469949635
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 83.23634974536745
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 74.59524897099772,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 55.606607254772754
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.59524897099772
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 58.27807685955187,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 65.89806036588965
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 58.27807685955187
      },
      "tr": {
       "accuracy": 1.0,
       "cc": 97.24435340666003,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 91.73306021998008
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 97.24435340666003
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 74.51178077969244,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 55.79340685520635
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 74.51178077969244
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 49.78741886942854,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 16.0289232749523
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 49.78741886942854
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.892,
       "cc": 86.05024971775197,
       "cc_by_type": {
        "choice": 86.05024971775197
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 86.05024971775197
      },
      "noul": {
       "accuracy": 0.9146666666666666,
       "cc": 82.93333333333332,
       "cc_by_type": {
        "noul": 82.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 82.93333333333332
      },
      "score": {
       "accuracy": 0.8026666666666666,
       "cc": 69.84443725780446,
       "cc_by_type": {
        "score": 69.84443725780446
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 69.84443725780446
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 51.71841212479776,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 50.0,
        "score": 37.155236374393255
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 51.71841212479776
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 80.19260915189135,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100.0,
        "score": 77.84490820101566
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.19260915189135
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 62.18230459928089,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": 35.32740160272074
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.18230459928089
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 35.481357551990506,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": 31.590271486380892
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 35.481357551990506
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 60.346106569192706,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 76.69049362062161
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 60.346106569192706
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 27.905726525665614,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -57.79225438526731
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 27.905726525665614
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 82.01582185942733,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 46.047465578282
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 82.01582185942733
      },
      "gaming": {
       "accuracy": 0.8181818181818182,
       "cc": 67.86827777219527,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50.0,
        "score": 64.05779498905969
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 67.86827777219527
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 61.15211571169912,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 85.95634713509736
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 61.15211571169912
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 76.09635878356498,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100.0,
        "score": 53.98500782821096
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 76.09635878356498
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 75.32488161969717,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 59.824774058057905
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 75.32488161969717
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 35.17291303717321,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 36.70153481044436
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 35.17291303717321
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 73.65620369970586,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 72.4849258783882
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 73.65620369970586
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 79.42571605349265,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 100.0,
        "score": 47.62779751112731
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.42571605349265
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 17.821719144102786,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 88.01410177012026
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 17.821719144102786
      },
      "recruiting": {
       "accuracy": 0.9444444444444444,
       "cc": 74.39779570718791,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 89.86005378823037
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 74.39779570718791
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 70.90997418649607,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 100.0,
        "score": 27.508247682640942
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 70.90997418649607
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 36.42130234970749,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 52.742167918687684
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 36.42130234970749
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.82840558406987,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.48521675220962
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.82840558406987
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.77948599085776,
      91.12581540138535
     ],
     "I": [
      75.46574808547649,
      82.6119466329837
     ],
     "capability": [
      81.10066647680395,
      86.42295095099455
     ],
     "composite": {
      "A": [
       71.77822793115342,
       73.87651780604975
      ],
      "B": [
       72.53976200330692,
       75.43964488054165
      ],
      "C": [
       71.77822793115342,
       73.87651780604975
      ]
     },
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.37957627728649,
     "capability": 84.33880376101371,
     "intelligence": 79.29803124474094
    },
    "per_type": {
     "choice": {
      "accuracy": 0.892,
      "calibration": 92.33099766610799,
      "cc": 86.05024971775197,
      "cc_by_type": {
       "choice": 86.05024971775197
      },
      "cc_tiered": 85.58910044641468,
      "cc_tiered_open": 85.05319626115181,
      "cc_tiered_sealed": 86.12500463167756,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 86.05024971775197
     },
     "noul": {
      "accuracy": 0.9146666666666666,
      "calibration": 87.97883267137907,
      "cc": 82.93333333333332,
      "cc_by_type": {
       "noul": 82.93333333333332
      },
      "cc_tiered": 83.36747203234482,
      "cc_tiered_open": 85.29951690821257,
      "cc_tiered_sealed": 81.43542715647708,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 82.93333333333332
     },
     "score": {
      "accuracy": 0.8026666666666666,
      "calibration": 87.82889849437241,
      "cc": 69.84443725780446,
      "cc_by_type": {
       "score": 69.84443725780446
      },
      "cc_tiered": 68.93752125546332,
      "cc_tiered_open": 69.57698909187614,
      "cc_tiered_sealed": 68.2980534190505,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 69.84443725780446
     }
    },
    "raw_on_A": {
     "calibration": 85.46585036403003,
     "capability": 81.17415269279076,
     "intelligence": 76.88245502155146
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.87634143842183,
    "Emphasis on Accuracy 60:20:20": 72.69805859484804,
    "Emphasis on Cost 20:20:60": 60.166025128160896,
    "Emphasis on Speed 20:60:20": 75.87184847359687,
    "Intelligence only": 79.29803124474094
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "H2O.ai",
   "axes": {
    "calibration": 89.95399367673842,
    "cost": 64.04698652895101,
    "intelligence": 60.03839906196252,
    "speed": 92.37207479774074
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03794010455551056,
      "ece_score": 92.4119790888979,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 92.4119790888979,
      "score_before_time_limit": 92.4119790888979,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.1156787859671944,
      "ece": 0.055451181905507255,
      "n": 375,
      "score": 88.90976361889855,
      "score_before_time_limit": 88.90976361889855,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.952,
      "interval90_width": 0.45600000000000007,
      "n": 375,
      "nrps": 0.08555524714180213,
      "score": 88.54023832241884,
      "score_before_time_limit": 88.54023832241884,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0718199932049105
     }
    },
    "score": 89.95399367673842
   },
   "capability": 74.99619636935047,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     70.96215547731991,
     75.09472546776658
    ],
    "B": [
     66.9048175236782,
     72.52438422150867
    ],
    "C": [
     58.560650129494064,
     75.09472546776658
    ]
   },
   "cost": {
    "basis": "ESTIMATE (self-hosted open weights, no public tariff): cost per 1,000 decisions measured for H2O-Lightning-4B v1.0 on the v1.5 pool (854,867 input tokens over 1,624 decisions, 0 output tokens; same base, shim and token path) carried for v1.1, the practice of every Qwen3.5-4B row on this board, at USD 0.03 per 1M input and USD 0.15 per 1M output (reference deepinfra:Qwen/Qwen3.5-4B, frozen 25 Sep 2026 snapshot) = USD 0.015792 per 1,000 decisions. Cross-check: v1.1's own v1.6 input tokens on the common cost basis give USD 0.0153 per 1,000 (within 4 %). Corrected in v1.6.37, no new measurement: the earlier cost of this row used USD 0.04 per 1M input; the frozen reference for Qwen/Qwen3.5-4B is USD 0.03 per 1M input and USD 0.15 per 1M output (DeepInfra, 25 Sep 2026 snapshot).",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015791878078817736
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "H2O-Lightning-4B v1.1 (H2O.ai, Qwen3.5-4B fine-tune, stock vLLM + open shim)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-04",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX 5090, official vLLM 0.30.0 image + the author's open standard-library shim (h2o_lightning_shim.py) as published on the model card, offline (all credentials unset), driven over loopback only by the JevBench client; label-free inputs; pod destroyed afterwards; JevBench add-request (GitHub #181), measured 4 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 61.05316077664259,
    "I_sealed": 59.02363734728246,
    "base": 60.03839906196252,
    "excess": -0.5438642130781113,
    "gap": 2.0295234293601325,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.18196275852294,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 80.22412656558997,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 60.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 50.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 47.035490585897584,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.83317199906027,
       "hard": 55.09009420986275,
       "judge": 42.073705022183916,
       "standard": 34.47012097695641
      }
     },
     "sealed|choice": {
      "cc": 64.65667579231118,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 60.80316742081447,
       "judge": 67.41154562383613,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 68.08195642942992,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 55.33980582524271,
       "judge": 68.42105263157893,
       "standard": 83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 44.33227982010628,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.681000195734974,
       "hard": 40.36020718747171,
       "judge": 45.957875906717405,
       "standard": 40.163670767644376
      }
     }
    }
   },
   "jevbench_score": 73.78423785625786,
   "key": "h2o-lightning-4b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "h2oai/h2o-lightning-4b@193ad740925b176a3b70a5a13a7cff2f2fadd01e",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.848,
    "decisive_rate": 1.0,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.95399367673842,
     "cost": 64.04698652895101,
     "intelligence": 60.03839906196252,
     "speed": 92.61084474720658
    },
    "board_release": "v1.6.43",
    "capability": 74.99619636935047,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 13,
    "capability_rank": 24,
    "composite_ci95": {
     "A": [
      70.99731037439084,
      75.13409530573585
     ],
     "B": [
      66.92981430724642,
      72.55375740934666
     ],
     "C": [
      58.58968210866478,
      75.13409530573585
     ]
    },
    "jevbench_score": 73.82224524014528,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 2,
    "rank": 10,
    "ranks": {
     "A": 10,
     "B": 18,
     "C": 10,
     "capability": 44
    },
    "scores": {
     "A": 73.82224524014528,
     "B": 70.5813765538927,
     "C": 73.82224524014528
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20856922594830393,
     "p50_s_raw": 0.02928461297415197,
     "p95_s_adjusted": 0.2628241665661335,
     "p95_s_raw": 0.056412083283066745
    },
    "views": {
     "Balanced 33:33:33": 69.65823125473824,
     "Emphasis on Accuracy 60:20:20": 65.4626515419746,
     "Emphasis on Cost 20:20:60": 67.2997420088874,
     "Emphasis on Speed 20:60:20": 77.3237822737329,
     "Intelligence only": 60.038399061962515
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a8 (scorer output sha256 7a5b4d4c7b7c, method option B / O1S)"
   },
   "rank": 11,
   "ranked": true,
   "ranks": {
    "A": 11,
    "B": 20,
    "C": 11,
    "capability": 56
   },
   "repo": "https://huggingface.co/h2oai/h2o-lightning-4b",
   "repriced_in": "v1.6.37",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.78423785625786,
    "B": 70.55357837863741,
    "C": 73.78423785625786
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21219652900472283,
    "p50_s_raw": 0.031098264502361417,
    "p95_s_adjusted": 0.2729318660800345,
    "p95_s_raw": 0.06146593304001726
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "supersedes_base_row": "v1.6.1",
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.64761887040858,
      90.96540455486856
     ],
     "I": [
      54.48601679476801,
      63.869097672212845
     ],
     "capability": [
      70.2236559039794,
      76.81358157643643
     ],
     "composite": {
      "A": [
       70.96215547731991,
       75.09472546776658
      ],
      "B": [
       66.9048175236782,
       72.52438422150867
      ],
      "C": [
       58.560650129494064,
       75.09472546776658
      ]
     },
     "seed": 44
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.95399367673842,
     "capability": 74.99619636935047,
     "intelligence": 60.03839906196252
    },
    "per_type": {
     "choice": {
      "accuracy": 0.744,
      "calibration": 92.4119790888979,
      "cc": 67.26125631139698,
      "cc_by_type": {
       "choice": 67.26125631139698
      },
      "cc_tiered": 69.91931927541705,
      "cc_tiered_open": 75.18196275852294,
      "cc_tiered_sealed": 64.65667579231118,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.26125631139698
     },
     "noul": {
      "accuracy": 0.848,
      "calibration": 88.90976361889855,
      "cc": 69.6,
      "cc_by_type": {
       "noul": 69.6
      },
      "cc_tiered": 64.51199270746858,
      "cc_tiered_open": 60.94202898550725,
      "cc_tiered_sealed": 68.08195642942992,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.6
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 88.54023832241884,
      "cc": 45.79426376408395,
      "cc_by_type": {
       "score": 45.79426376408395
      },
      "cc_tiered": 45.683885203001935,
      "cc_tiered_open": 47.035490585897584,
      "cc_tiered_sealed": 44.33227982010628,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 45.79426376408395
     }
    },
    "raw_on_A": {
     "calibration": 86.31592326935305,
     "capability": 74.50666727956846,
     "intelligence": 62.69741128978386
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.61311641822417,
    "Emphasis on Accuracy 60:20:20": 65.43873845576302,
    "Emphasis on Cost 20:20:60": 67.27446819449806,
    "Emphasis on Speed 20:60:20": 77.22378399112957,
    "Intelligence only": 60.038399061962515
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Hyland",
   "axes": {
    "calibration": 75.8316875668352,
    "cost": 49.09977911142268,
    "intelligence": 52.865450899914414,
    "speed": 93.51379976299526
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14317514920634938,
      "ece_score": 71.36497015873012,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 70.41343722328038,
      "score_before_time_limit": 71.36497015873012,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.16800000000000004,
      "brier": 0.12771623160380524,
      "ece": 0.09925250399999994,
      "n": 375,
      "score": 79.72203520426667,
      "score_before_time_limit": 80.14949920000001,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9066666666666666,
      "interval90_width": 0.3264444444444444,
      "n": 375,
      "nrps": 0.09940238260432994,
      "score": 77.35959027295856,
      "score_before_time_limit": 77.98345793645015,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.1704642293333334
     }
    },
    "score": 75.8316875668352
   },
   "capability": 64.3485692333748,
   "composite_ci95": {
    "A": [
     52.374566728540906,
     62.42181266526091
    ],
    "B": [
     49.592754478501206,
     60.69955947070597
    ],
    "C": [
     36.37122689482008,
     55.479920762925325
    ]
   },
   "cost": {
    "basis": "ESTIMATE, v1.5-M2 base-model reference rule. The customer states a benchmark tariff of USD 0.056 per million input tokens, output free (one NVIDIA L40S, AWS on-demand); there is no public price page for it, so it is not a published list price. Under the method, the base-model reference price applies as an ESTIMATE for this vendor-hosted row, as for attest-31b-v1.0: base model google/gemma-4-26b-a4b-it through the frozen pricing_v15 module (sha256 1f4d1f7b1ce97ab27b938043444b9c2757e4a9dbecc799a803122588401f5bf1) reference() -> google/gemma-4-26b-a4b-it on OpenRouter, USD 0.09 per million input tokens, USD 0.30 per million output tokens. The customer-stated tariff is shown for information and is not used for the Cost axis. Applied to this row's own provider-reported tokens, first answer per item (2,012,340 input and 0 output tokens over the 1,500 items; 820,650 input and 0 output tokens over the 1,485 items of the common cost basis; the 15 very long items count for no row's cost, whether the row refused or answered them). Estimated, not a vendor bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them in full).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.1207404
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04973636363636364
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "HS1 0.1.2 (Hyland)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane order 880b09e8. Vendor-hosted Hyland API (customer-supplied HTTPS endpoint, model `hyland-hs1`), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) over the official per-item latency subset of the first-occurrence rows, without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `hyland` (no proxy) before the first request. 1,500 of 1,500 unique items answered with HTTP 200 and the served model was `hyland-hs1` on every row; option-letter log-probability readout, nothing generated (0 output tokens). Run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 60.80592519588509,
    "I_sealed": 44.92497660394374,
    "base": 52.865450899914414,
    "excess": 6.041421821585637,
    "gap": 15.880948591941348,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 64.83372028845844,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 60.36489462094998,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 61.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 19.999999999999996,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 56.46328235233694,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.266396969696956,
       "hard": 57.13894351851854,
       "judge": 57.104464406779655,
       "standard": 54.24862962962963
      }
     },
     "sealed|choice": {
      "cc": 63.179889701235524,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 79.03430749682339,
       "hard": 56.80582861107898,
       "judge": 62.09598432908915,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 36.31329247181023,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 34.54545454545455,
       "judge": 19.999999999999996,
       "standard": 46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 35.281747638785475,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 62.096657894736836,
       "hard": 26.543902535211284,
       "judge": 26.121538541666666,
       "standard": 53.09029636363637
      }
     }
    }
   },
   "jevbench_score": 61.068034493527705,
   "key": "hyland-hs1",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary hosted API (no weights release); declared base Gemma 4 26B-A4B-it is under the Gemma 4 licence",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:hyland.com/hyland-hs1/v1/systemone#hyland-hs1",
   "noul_decisive": {
    "acc_among_decisive": 0.8838709677419355,
    "decisive": 310,
    "decisive_rate": 0.8310991957104558,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 24,
    "axes": {
     "calibration": 76.49930909839343,
     "cost": 49.09977911142268,
     "intelligence": 53.57948799459325,
     "speed": 93.74328417303443
    },
    "board_release": "v1.6.43",
    "capability": 65.03939854649335,
    "capability_api_board_rank": 21,
    "capability_open_board_rank": null,
    "capability_rank": 59,
    "composite_ci95": {
     "A": [
      54.8992123734169,
      62.833909581191186
     ],
     "B": [
      52.24566392674325,
      61.163125168542486
     ],
     "C": [
      38.12445303709507,
      57.62480456059248
     ]
    },
    "jevbench_score": 61.450719939462154,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 56,
    "ranks": {
     "A": 56,
     "B": 54,
     "C": 53,
     "capability": 101
    },
    "scores": {
     "A": 61.450719939462154,
     "B": 59.20850453421004,
     "C": 49.002878617843415
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.19411396997747943,
     "p50_s_raw": 0.19411396997747943,
     "p95_s_adjusted": 0.21757790509145708,
     "p95_s_raw": 0.21757790509145708
    },
    "views": {
     "Balanced 33:33:33": 58.21053424780427,
     "Emphasis on Accuracy 60:20:20": 55.404055858567006,
     "Emphasis on Cost 20:20:60": 53.31752802618845,
     "Emphasis on Speed 20:60:20": 67.87813214465525,
     "Intelligence only": 51.66752129933851
    }
   },
   "rank": 63,
   "ranked": true,
   "ranks": {
    "A": 63,
    "B": 62,
    "C": 64,
    "capability": 120
   },
   "repo": "https://www.hyland.com",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 61.068034493527705,
    "B": 58.74291323238438,
    "C": 47.40840267556635
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19862511445535347,
    "p50_s_raw": 0.19862511445535347,
    "p95_s_adjusted": 0.2241743211168796,
    "p95_s_raw": 0.2241743211168796
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no customer code ran locally; weights and serving code are not public. Customer statements (not independently verified): HS1 0.1.2 is a LoRA fine-tune of Gemma-4-26B-A4B-it served in FP8 on one NVIDIA L40S (AWS on-demand); trained on \"10,762 curated rows\". DUPLICATE REQUESTS, FIRST ANSWER SCORED: because of an operator error two dispatches of the same driver overlapped briefly, so 64 items (items 2 to 65 of the run order) were requested twice; the first answer per item is scored, the 64 repeat rows are not scored, and the repeats carry no new exposure beyond the admitted full-set run. FULL LENGTH: the endpoint accepted every item, including the 15 longest (reported input 76,572 to 81,749 tokens), within its 131,072-token context; nothing was refused or truncated. MODEL, customer statements (unverified): LoRA fine-tune of Gemma-4-26B-A4B-it served in FP8; training on \"10,762 curated rows\"; weights and code are not public (API entrant). PUBLIC ITEMS: the customer was asked whether public JevBench items were used in training or model selection and had not answered when this release was built, so the normal public-vs-sealed gap rule applies without any claim either way.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 67.35632222222223,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 10.402299999999997
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 67.35632222222223
      },
      "calibration_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 46.96054865897844,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 60.00000000000001,
        "score": 15.876343749999988
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 46.96054865897844
      },
      "composite_gen": {
       "accuracy": 0.6732673267326733,
       "cc": 52.270568278785355,
       "cc_by_type": {
        "choice": 77.23577235772358,
        "noul": 55.55555555555556,
        "score": 24.020376923076913
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 52.270568278785355
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 80.64695185185185,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 81.94085555555557
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 80.64695185185185
      },
      "jaggedness": {
       "accuracy": 0.5277777777777778,
       "cc": 26.260480774082126,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": 2.7027027027026973,
        "score": 30.939182352941174
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 26.260480774082126
      },
      "llm_router": {
       "accuracy": 0.7859424920127795,
       "cc": 69.77902567381767,
       "cc_by_type": {
        "choice": 66.71656247248393,
        "noul": 79.38144329896907,
        "score": 63.23907124999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.77902567381767
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5925925925925926,
       "cc": 29.34662629684176,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": -25.0,
        "score": 50.15328095238094
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 29.34662629684176
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 50.017188940458446,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": 36.111111111111114,
        "score": 52.08848309859155
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 50.017188940458446
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 39.70408444444444,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 19.999999999999996,
        "score": 5.7789199999999985
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 39.70408444444444
      },
      "option_order": {
       "accuracy": 0.75,
       "cc": 68.04260985352863,
       "cc_by_type": {
        "choice": 68.04260985352863
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 68.04260985352863
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 79.1600528105028,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100.0,
        "score": 67.51018846153846
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.1600528105028
      },
      "usecase": {
       "accuracy": 0.6,
       "cc": 43.19437939607844,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": 29.166666666666675,
        "score": 38.40666760000001
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 43.19437939607844
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 36.60442454212454,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": 16.95613076923076
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 36.60442454212454
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 45.20500871479774,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 23.01901999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 45.20500871479774
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 77.65046457611902,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 100.0,
        "score": 80.7390043478261
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.65046457611902
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 38.651429019242705,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 99.56694999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 38.651429019242705
      },
      "en": {
       "accuracy": 0.6794092093831451,
       "cc": 50.40503834349743,
       "cc_by_type": {
        "choice": 64.63308966207391,
        "noul": 47.482014388489205,
        "score": 39.100010979929166
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 50.40503834349743
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 53.85744628794422,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100.0,
        "score": -9.063876470588216
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 53.85744628794422
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.90507650273224,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 99.95566666666666
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.90507650273224
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 67.4717750257998,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100.0,
        "score": 65.94473684210527
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 67.4717750257998
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 57.72511666666666,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 25.0,
        "score": 48.17534999999998
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 57.72511666666666
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 60.872964135396934,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 10.678128571428546
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.872964135396934
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 35.41934512046597,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 39.99999999999999,
        "score": 23.446279999999998
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 35.41934512046597
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 38.989697089947086,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 61.41353571428571
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 38.989697089947086
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.76121111111111,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.9503
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.76121111111111
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 27.763858239538234,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 68.57296
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 27.763858239538234
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 45.71841414900911,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 19.999999999999996,
        "score": 40.00754883720929
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.71841414900911
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 72.995288488224,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 50.8067258064516
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.995288488224
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 22.163285106382986,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 23.085600000000017
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 22.163285106382986
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 55.01890000000001,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 15.056700000000035
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 55.01890000000001
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.50802365591399,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 91.04020000000001
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.50802365591399
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 33.86654595103578,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": 60.3567
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.86654595103578
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7253333333333334,
       "cc": 64.52285730687538,
       "cc_by_type": {
        "choice": 64.52285730687538
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 64.52285730687538
      },
      "noul": {
       "accuracy": 0.7306666666666667,
       "cc": 46.13333333333334,
       "cc_by_type": {
        "noul": 46.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 46.13333333333334
      },
      "score": {
       "accuracy": 0.5466666666666666,
       "cc": 41.569475090252716,
       "cc_by_type": {
        "score": 41.569475090252716
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 41.569475090252716
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -11.300584313725494,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -50.0,
        "score": -19.901752941176486
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 81.0909294871795,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 43.27278846153847
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 81.0909294871795
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 18.76733739837398,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": -44.91750000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 18.76733739837398
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 0.1842885233918056,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -33.333333333333336,
        "score": 4.061637499999982
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.1842885233918056
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 35.30437397342996,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 68.23196250000001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 35.30437397342996
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -55.67849392033543,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 0.0,
        "score": -204.1424
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 39.19570028729439,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 32.713151282051264
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 39.19570028729439
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 5.0591438569913665,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -100.0,
        "score": 56.98927826086957
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 5.0591438569913665
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 13.626879629629627,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 19.999999999999996,
        "score": 4.213972222222218
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 13.626879629629627
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 46.05044599095885,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 35.72450000000001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 46.05044599095885
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 44.080392655234526,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 55.55555555555556,
        "score": -0.058563636363628824
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 44.080392655234526
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": 16.414429256272395,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 44.94221249999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 16.414429256272395
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 57.32517348048626,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 35.00814999999999
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 57.32517348048626
      },
      "model_routing": {
       "accuracy": 0.8387096774193549,
       "cc": 75.62815107097764,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 93.93939393939394,
        "score": 53.9840203125
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.62815107097764
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 16.569089191223515,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 61.22358235294117
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 16.569089191223515
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 51.46014982977406,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 54.473300000000016
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 51.46014982977406
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 16.829311600502272,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 0.0,
        "score": 9.60123529411765
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 16.829311600502272
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 25.47528971014493,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 53.23746333333334
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 25.47528971014493
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 93.97886359493039,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 92.61274736842105
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 93.97886359493039
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.79674006801244,
      78.35115191592271
     ],
     "I": [
      47.34948055006876,
      56.61223115738562
     ],
     "capability": [
      59.653393276541806,
      66.81231879137462
     ],
     "composite": {
      "A": [
       52.374566728540906,
       62.42181266526091
      ],
      "B": [
       49.592754478501206,
       60.69955947070597
      ],
      "C": [
       36.37122689482008,
       55.479920762925325
      ]
     },
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.8316875668352,
     "capability": 64.3485692333748,
     "intelligence": 52.865450899914414
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7253333333333334,
      "calibration": 70.41343722328038,
      "cc": 64.52285730687538,
      "cc_by_type": {
       "choice": 64.52285730687538
      },
      "cc_tiered": 64.00680499484699,
      "cc_tiered_open": 64.83372028845844,
      "cc_tiered_sealed": 63.179889701235524,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 64.52285730687538
     },
     "noul": {
      "accuracy": 0.7306666666666667,
      "calibration": 79.72203520426667,
      "cc": 46.13333333333334,
      "cc_by_type": {
       "noul": 46.13333333333334
      },
      "cc_tiered": 48.717032709335065,
      "cc_tiered_open": 61.1207729468599,
      "cc_tiered_sealed": 36.31329247181023,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.13333333333334
     },
     "score": {
      "accuracy": 0.5466666666666666,
      "calibration": 77.35959027295856,
      "cc": 41.569475090252716,
      "cc_by_type": {
       "score": 41.569475090252716
      },
      "cc_tiered": 45.872514995561204,
      "cc_tiered_open": 56.46328235233694,
      "cc_tiered_sealed": 35.281747638785475,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.569475090252716
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 57.88642123418364,
    "Emphasis on Accuracy 60:20:20": 54.91036265742412,
    "Emphasis on Cost 20:20:60": 53.153968766440535,
    "Emphasis on Speed 20:60:20": 67.56365834205323,
    "Intelligence only": 50.97896438738072
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "mohit67890",
   "axes": {
    "calibration": 86.77653179414553,
    "cost": 51.76166122103225,
    "intelligence": 21.160522003324633,
    "speed": 81.2429913982843
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03246566765774192,
      "ece_score": 93.5068664684516,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 93.5068664684516,
      "score_before_time_limit": 93.5068664684516,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.6246648793565683,
      "brier": 0.2028168457960074,
      "ece": 0.11038035259514468,
      "n": 373,
      "score": 77.92392948097107,
      "score_before_time_limit": 77.92392948097107,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9408602150537635,
      "interval90_width": 0.6883960573476698,
      "n": 372,
      "nrps": 0.1322491683106928,
      "score": 88.89879943301392,
      "score_before_time_limit": 88.89879943301392,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.044887421514514296
     }
    },
    "score": 86.77653179414553
   },
   "capability": 53.96852689873508,
   "composite_ci95": {
    "A": [
     4.656344644025745,
     10.934355358595067
    ],
    "B": [
     3.7056232421260704,
     9.202876545120338
    ],
    "C": [
     3.233572669462324,
     7.593302332357687
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-2B -> deepinfra:Qwen/Qwen3.5-2B USD 0.02/M input, USD 0.1/M output. Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835, the module this row was first priced with (the later re-freezes only add hosted-API launch-list keys), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). Tokens: input = 4-pass sum counted offline (see disclosures), output 0. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.040546 per 1,000 decisions (0.628x the official cost cap USD 0.064595), against USD 0.040140 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.040140226666666674
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.040545683501683506
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Imajev-2B (mohit67890, Qwen3.5-2B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20150, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own scripts/playground/server.py --backend torch, 4 option orders, 4,096-token request limit.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 24.18657872532247,
    "I_sealed": 18.134465281326797,
    "base": 21.160522003324633,
    "excess": -3.7874133263600385,
    "gap": 6.0521134439956725,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 49.471301249374704,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 37.71626297577854,
       "judge": 52.537903757415954,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -42.85714285714286,
       "hard": -13.043478260869568,
       "judge": -60.0,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 23.08843492659271,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.907855488528355,
       "hard": 14.056431441245543,
       "judge": 33.77791221517532,
       "standard": 8.208515683445306
      }
     },
     "sealed|choice": {
      "cc": 41.84862177374901,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 54.2566709021601,
       "hard": 42.23430091361164,
       "judge": 38.29578844270325,
       "standard": 40.20248892638684
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -27.586206896551722,
       "hard": -49.09090909090909,
       "judge": -63.07692307692307,
       "standard": -7.462686567164178
      }
     },
     "sealed|score": {
      "cc": 12.554774070231389,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 26.48562308030278,
       "hard": 9.945290312389155,
       "judge": 9.940140408699815,
       "standard": 14.730267573177514
      }
     }
    }
   },
   "jevbench_score": 7.924238221366574,
   "key": "imajev-2b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0 (adapter; base Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "mohit67890/imajev-2b@66e4b8c808bae45ca98bed5edd2de6323bdb0576",
   "noul_decisive": {
    "acc_among_decisive": 0.8428571428571429,
    "decisive": 140,
    "decisive_rate": 0.3753351206434316,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.77653179414553,
     "cost": 51.76166122103225,
     "intelligence": 21.160522003324633,
     "speed": 81.55302418750827
    },
    "board_release": "v1.6.43",
    "capability": 53.96852689873508,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 87,
    "capability_rank": 113,
    "composite_ci95": {
     "A": [
      4.658485929408603,
      10.94035382726261
     ],
     "B": [
      3.706707978661138,
      9.206275249274212
     ],
     "C": [
      3.2350596732004195,
      7.5974679355990355
     ]
    },
    "jevbench_score": 7.92834166805135,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 138,
    "rank": 177,
    "ranks": {
     "A": 177,
     "B": 178,
     "C": 179,
     "capability": 168
    },
    "scores": {
     "A": 7.92834166805135,
     "B": 6.507269614912114,
     "C": 5.50579282503566
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7288981560035609,
     "p50_s_raw": 0.28944907800178044,
     "p95_s_adjusted": 0.9594685189804295,
     "p95_s_raw": 0.40473425949021474
    },
    "views": {
     "Balanced 33:33:33": 6.815416220156846,
     "Emphasis on Accuracy 60:20:20": 5.165912913746609,
     "Emphasis on Cost 20:20:60": 7.623022614150632,
     "Emphasis on Speed 20:60:20": 8.663982835133574,
     "Intelligence only": 3.789999234949207
    }
   },
   "rank": 199,
   "ranked": true,
   "ranks": {
    "A": 199,
    "B": 200,
    "C": 201,
    "capability": 188
   },
   "repo": "https://huggingface.co/mohit67890/imajev-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 7.924238221366574,
    "B": 6.505057799589012,
    "C": 5.502943209282343
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.745621624001069,
    "p50_s_raw": 0.2978108120005345,
    "p95_s_adjusted": 1.0073545804945752,
    "p95_s_raw": 0.42867729024728757
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 27.940174431020967,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 50.0,
        "score": -32.846143373603766
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 27.940174431020967
      },
      "calibration_gen": {
       "accuracy": 0.297029702970297,
       "cc": -17.5634975525963,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -60.0,
        "score": -6.7944163057210405
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.43564356435643564,
       "cc": -4.661934044854566,
       "cc_by_type": {
        "choice": 38.211382113821145,
        "noul": -62.96296296296296,
        "score": 10.765778714578122
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.6458333333333334,
       "cc": 52.590783388621844,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 37.772350165865554
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 52.590783388621844
      },
      "jaggedness": {
       "accuracy": 0.3277777777777778,
       "cc": -7.850740882891453,
       "cc_by_type": {
        "choice": 17.70933589990377,
        "noul": -40.54054054054054,
        "score": -0.7210180080375883
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.5175718849840255,
       "cc": 18.658985210953194,
       "cc_by_type": {
        "choice": 51.65976930527428,
        "noul": -23.71134020618557,
        "score": 28.02852653377088
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 18.658985210953194
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -18.44398052388307,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75.0,
        "score": -6.105137447937858
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5309090909090909,
       "cc": 16.31443135528656,
       "cc_by_type": {
        "choice": 56.9611998695794,
        "noul": -36.111111111111114,
        "score": 28.09320530739138
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 16.31443135528656
      },
      "negation_pair_gen": {
       "accuracy": 0.36046511627906974,
       "cc": -14.889109686015951,
       "cc_by_type": {
        "choice": 29.999999999999996,
        "noul": -73.33333333333334,
        "score": -1.3339957247145096
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.5,
       "cc": 36.08521970705726,
       "cc_by_type": {
        "choice": 36.08521970705726
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 36.08521970705726
      },
      "perturb_pair": {
       "accuracy": 0.6875,
       "cc": 52.302256042037875,
       "cc_by_type": {
        "choice": 57.95795795795795,
        "noul": 39.99999999999999,
        "score": 58.948810168155674
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 52.302256042037875
      },
      "usecase": {
       "accuracy": 0.4727272727272727,
       "cc": 8.637486944515624,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": -45.833333333333336,
        "score": 18.31442161786059
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 8.637486944515624
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -28.225026395455945,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50.0,
        "score": -58.48460299589164
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 24.065748870216918,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 26.267907132924172
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 24.065748870216918
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 42.362310149525264,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 0.0,
        "score": 43.016133988398806
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 42.362310149525264
      },
      "el": {
       "accuracy": 0.4,
       "cc": 19.38235434668214,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 91.85283585196449
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 19.38235434668214
      },
      "en": {
       "accuracy": 0.44917463075586445,
       "cc": 5.074350761629012,
       "cc_by_type": {
        "choice": 40.24969185145407,
        "noul": -38.12949640287769,
        "score": 13.10285683631066
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 5.074350761629012
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 46.5562811978586,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 50.0,
        "score": 19.032628259154937
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 46.5562811978586
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -10.63743944027403,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": 33.661452170981185
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 3.2263733482564874,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": -100.0,
        "score": 52.03206122124004
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 3.2263733482564874
      },
      "hi": {
       "accuracy": 0.42857142857142855,
       "cc": -10.475686348831847,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100.0,
        "score": 18.852270562443007
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.4666666666666667,
       "cc": -19.711858509492984,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": -100.0,
        "score": -31.076339363241235
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 34.29447363096327,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 25.77460437184137
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 34.29447363096327
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -9.900639833853115,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 14.742524942885105
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.6428571428571429,
       "cc": 58.50507211449983,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 58.84854967683284
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 58.50507211449983
      },
      "nl": {
       "accuracy": 0.4,
       "cc": 3.7552735572627327,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -33.333333333333336,
        "score": 22.521231927199448
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 3.7552735572627327
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5128205128205128,
       "cc": 16.728251852982496,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40.0,
        "score": 13.03706194912947
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 16.728251852982496
      },
      "pt": {
       "accuracy": 0.39285714285714285,
       "cc": 20.212949326324164,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 19.999999999999996,
        "score": 14.887522109791584
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 20.212949326324164
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -0.25603227399320855,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -100.0,
        "score": 50.29573296525442
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 28.348191067910236,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50.0,
        "score": 0.2619645080785271
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 28.348191067910236
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 3.1976636096176585,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 74.10911986111104
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 3.1976636096176585
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 8.401209830979823,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": 9.384420453391451
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 8.401209830979823
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5666666666666667,
       "cc": 44.02877973172085,
       "cc_by_type": {
        "choice": 44.02877973172085
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 44.02877973172085
      },
      "noul": {
       "accuracy": 0.31466666666666665,
       "cc": -37.06666666666667,
       "cc_by_type": {
        "noul": -37.06666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.42133333333333334,
       "cc": 15.622459826305535,
       "cc_by_type": {
        "score": 15.622459826305535
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 15.622459826305535
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.09090909090909091,
       "cc": -33.193862534650656,
       "cc_by_type": {
        "choice": 4.0,
        "noul": -100.0,
        "score": -3.5815876039519523
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": -6.089719732637479,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -55.55555555555556,
        "score": -6.8129825243444575
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 62.03940267844437,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 36.11820803533312
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.03940267844437
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -37.42857709364494,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -30.414386251695056
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 10.87022836857408,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 9.422279308620796
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 10.87022836857408
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -8.016826814669251,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -33.333333333333336,
        "score": -2.6668326452656155
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 9.059322026345937,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 28.018302213491587
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 9.059322026345937
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -15.154690545329686,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 27.706660071328017
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -11.501806643589282,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -100.0,
        "score": 27.994580069232157
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": -10.82046302383224,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -33.333333333333336,
        "score": -9.192295566857167
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.44,
       "cc": -1.191894960640904,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -77.77777777777779,
        "score": -14.170000127400728
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -25.461994188704736,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -11.869853533856144
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.37037037037037035,
       "cc": -0.9809923099182641,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60.0,
        "score": 3.1222821873277407
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.5576036866359447,
       "cc": 21.130055825784556,
       "cc_by_type": {
        "choice": 63.76623376623377,
        "noul": -24.242424242424242,
        "score": 23.86635795354414
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 21.130055825784556
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -13.619363650781176,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": 16.72348294400964
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 53.4817305604329,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 43.824950269970955
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 53.4817305604329
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -1.6382323766226665,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -100.0,
        "score": 24.64195311643741
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 9.317909661738637,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 33.75083043449128
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 9.317909661738637
      },
      "search_retrieval": {
       "accuracy": 0.5833333333333334,
       "cc": 21.221925502780437,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": -50.0,
        "score": 45.694246259230994
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 21.221925502780437
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.44171405018487,
      86.64641382932699
     ],
     "I": [
      17.21228402122313,
      24.114657957043782
     ],
     "capability": [
      49.4688678734668,
      54.186102023350564
     ],
     "composite": {
      "A": [
       4.656344644025745,
       10.934355358595067
      ],
      "B": [
       3.7056232421260704,
       9.202876545120338
      ],
      "C": [
       3.233572669462324,
       7.593302332357687
      ]
     },
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.77653179414553,
     "capability": 53.96852689873508,
     "intelligence": 21.160522003324633
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5666666666666667,
      "calibration": 93.5068664684516,
      "cc": 44.02877973172085,
      "cc_by_type": {
       "choice": 44.02877973172085
      },
      "cc_tiered": 45.65996151156186,
      "cc_tiered_open": 49.471301249374704,
      "cc_tiered_sealed": 41.84862177374901,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 44.02877973172085
     },
     "noul": {
      "accuracy": 0.31466666666666665,
      "calibration": 77.92392948097107,
      "cc": -37.06666666666667,
      "cc_by_type": {
       "noul": -37.06666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.42133333333333334,
      "calibration": 88.89879943301392,
      "cc": 15.622459826305535,
      "cc_by_type": {
       "score": 15.622459826305535
      },
      "cc_tiered": 17.82160449841205,
      "cc_tiered_open": 23.08843492659271,
      "cc_tiered_sealed": 12.554774070231389,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 15.622459826305535
     }
    },
    "raw_on_A": {
     "calibration": 83.85821528875951,
     "capability": 52.090117601921726,
     "intelligence": 20.322019915083942
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.811373490230182,
    "Emphasis on Accuracy 60:20:20": 5.164518871771041,
    "Emphasis on Cost 20:20:60": 7.619987462702236,
    "Emphasis on Speed 20:60:20": 8.6522320766731,
    "Intelligence only": 3.789999234949207
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "mohit67890",
   "axes": {
    "calibration": 88.10935553643324,
    "cost": 30.792561090951693,
    "intelligence": 36.7327337780622,
    "speed": 79.23152050102456
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05597354899568825,
      "ece_score": 88.80529020086236,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.80529020086236,
      "score_before_time_limit": 88.80529020086236,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4584450402144772,
      "brier": 0.13920542517421816,
      "ece": 0.08183623766154813,
      "n": 373,
      "score": 83.63275246769038,
      "score_before_time_limit": 83.63275246769038,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.5192652329749106,
      "n": 372,
      "nrps": 0.09346155028027302,
      "score": 91.89002394074701,
      "score_before_time_limit": 91.89002394074701,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.034368985452393384
     }
    },
    "score": 88.10935553643324
   },
   "capability": 62.42104465724772,
   "composite_ci95": {
    "A": [
     7.169224741773472,
     12.715245634836354
    ],
    "B": [
     6.623658641306371,
     12.218997228813015
    ],
    "C": [
     4.978628292898244,
     8.830031690858576
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-9B -> qwen/qwen3.5-9b USD 0.09999999999999999/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835, the module this row was first priced with (the later re-freezes only add hosted-API launch-list keys), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). Tokens: input = 4-pass sum counted offline (see disclosures), output 0. OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.202728 per 1,000 decisions on the common basis (USD 0.200701 over all 1,500 items) is 3.138x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.202728 per 1,000 decisions (3.138x the official cost cap USD 0.064595), against USD 0.200701 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.20070113333333328
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.20272841750841747
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Imajev-9B (mohit67890, Qwen3.5-9B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20150, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own scripts/playground/server.py --backend torch, 4 option orders, 4,096-token request limit.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.23991434770264,
    "I_sealed": 32.22555320842176,
    "base": 36.7327337780622,
    "excess": -0.8251656310748317,
    "gap": 9.01436113928088,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 67.89294454901062,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 62.25228059138094,
       "judge": 72.31377719182598,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 13.632160110420985,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 21.739130434782616,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 42.19463838367629,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.06511169132162,
       "hard": 33.49205778853101,
       "judge": 43.74474924939493,
       "standard": 54.33939662156624
      }
     },
     "sealed|choice": {
      "cc": 61.96527273043063,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 54.72418179715507,
       "judge": 62.09598432908915,
       "standard": 70.57582788441259
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 20.68965517241379,
       "hard": -9.090909090909093,
       "judge": -38.46153846153846,
       "standard": 13.432835820895516
      }
     },
     "sealed|score": {
      "cc": 34.71138689483466,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 55.85407978179955,
       "hard": 30.051996837602136,
       "judge": 30.53981630846708,
       "standard": 39.716176445368625
      }
     }
    }
   },
   "jevbench_score": 9.786090999130614,
   "key": "imajev-9b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0 (adapter; base Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "mohit67890/imajev-9b@7301600591665b941d324b1a297e097b56724f9b",
   "noul_decisive": {
    "acc_among_decisive": 0.9257425742574258,
    "decisive": 202,
    "decisive_rate": 0.5415549597855228,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.10935553643324,
     "cost": 30.792561090951693,
     "intelligence": 36.7327337780622,
     "speed": 79.48842001125843
    },
    "board_release": "v1.6.43",
    "capability": 62.42104465724772,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      7.172547616858205,
      12.721677918306504
     ],
     "B": [
      6.6259340523666275,
      12.223727353008137
     ],
     "C": [
      4.980935845040419,
      8.834498554379516
     ]
    },
    "jevbench_score": 9.790864242743115,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 133,
    "rank": 172,
    "ranks": {
     "A": 172,
     "B": 170,
     "C": 173,
     "capability": 117
    },
    "scores": {
     "A": 9.790864242743115,
     "B": 9.23297819070672,
     "C": 6.7992112796827175
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.9162941940070596,
     "p50_s_raw": 0.3831470970035298,
     "p95_s_adjusted": 1.227787094612722,
     "p95_s_raw": 0.5388935473063611
    },
    "views": {
     "Balanced 33:33:33": 8.496190189886166,
     "Emphasis on Accuracy 60:20:20": 8.076430781084673,
     "Emphasis on Cost 20:20:60": 7.458272603841369,
     "Emphasis on Speed 20:60:20": 10.503863437577062,
     "Intelligence only": 7.519195208738335
    }
   },
   "rank": 194,
   "ranked": true,
   "ranks": {
    "A": 194,
    "B": 192,
    "C": 195,
    "capability": 134
   },
   "repo": "https://huggingface.co/mohit67890/imajev-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 9.786090999130614,
    "B": 9.229581960277148,
    "C": 6.795896527174036
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.9317987710295711,
    "p50_s_raw": 0.39089938551478554,
    "p95_s_adjusted": 1.2809312370838597,
    "p95_s_raw": 0.5654656185419299
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 52.430958850575394,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": -9.373790114940505
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 52.430958850575394
      },
      "calibration_gen": {
       "accuracy": 0.46534653465346537,
       "cc": 5.028160919179807,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": -36.0,
        "score": 8.34853365891799
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 5.028160919179807
      },
      "composite_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 19.016879555076702,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -48.14814814814815,
        "score": 37.7191120166303
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.016879555076702
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 77.21353987562888,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 71.64061962688662
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.21353987562888
      },
      "jaggedness": {
       "accuracy": 0.4777777777777778,
       "cc": 8.891936687850714,
       "cc_by_type": {
        "choice": 46.58325312800771,
        "noul": -51.35135135135135,
        "score": 31.443908286895773
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 8.891936687850714
      },
      "llm_router": {
       "accuracy": 0.7987220447284346,
       "cc": 62.159239624897964,
       "cc_by_type": {
        "choice": 76.22611605177423,
        "noul": 54.639175257731964,
        "score": 55.612427565187716
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.159239624897964
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -26.109429454822063,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100.0,
        "score": 8.269649779863697
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6327272727272727,
       "cc": 34.94749358869319,
       "cc_by_type": {
        "choice": 66.7427453537659,
        "noul": -8.333333333333337,
        "score": 46.43306874564702
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.94749358869319
      },
      "negation_pair_gen": {
       "accuracy": 0.46511627906976744,
       "cc": 0.2664008509962761,
       "cc_by_type": {
        "choice": 63.333333333333336,
        "noul": -66.66666666666667,
        "score": 4.132535886322164
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.2664008509962761
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 77.05660729696058,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100.0,
        "score": 55.19384591490575
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 77.05660729696058
      },
      "usecase": {
       "accuracy": 0.5590909090909091,
       "cc": 31.572158461704316,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 0.0,
        "score": 33.932161659622736
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 31.572158461704316
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -5.14602112844375,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0.0,
        "score": -39.24758719485506
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 40.62744831788267,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 30.023665998194872
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 40.62744831788267
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 65.36973340580671,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 50.0,
        "score": 62.03840375724315
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 65.36973340580671
      },
      "el": {
       "accuracy": 0.4,
       "cc": 15.425167488428194,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100.0,
        "score": 96.55483207422313
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 15.425167488428194
      },
      "en": {
       "accuracy": 0.6194613379669852,
       "cc": 32.65410958205473,
       "cc_by_type": {
        "choice": 63.53473840934329,
        "noul": -0.7194244604316502,
        "score": 35.14701479725255
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 32.65410958205473
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 62.56797666185319,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 7.279786429279
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.56797666185319
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 18.91749265853117,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -100.0,
        "score": 89.53936322149515
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.91749265853117
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 47.06376202766997,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 83.54422725948051
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 47.06376202766997
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 9.1729321161986,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -75.0,
        "score": 52.798125957534346
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 9.1729321161986
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 63.89055758345746,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 5.701290832991235
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.89055758345746
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 29.244543319970006,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 10.62481343886158
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.244543319970006
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 2.3136682734769445,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 51.385449264875284
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 2.3136682734769445
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.97684526925495,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.59720247443153
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.97684526925495
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 7.280856601680536,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 47.81659577906757
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.280856601680536
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -14.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 40.43577295910621,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0.0,
        "score": 36.542189804106606
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 40.43577295910621
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 48.520372352488316,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": 60.00000000000001,
        "score": 49.202837741024176
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 48.520372352488316
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 8.986079710155908,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 49.0858987049358
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 8.986079710155908
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 14.927301478010037,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": -11.739834696404671
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 14.927301478010037
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.27310580160078,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 58.81931740480236
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.27310580160078
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 19.69631508464502,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -7.577721412731608
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.69631508464502
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.72,
       "cc": 63.83398074972732,
       "cc_by_type": {
        "choice": 63.83398074972732
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 63.83398074972732
      },
      "noul": {
       "accuracy": 0.49866666666666665,
       "cc": -0.2666666666666706,
       "cc_by_type": {
        "noul": -0.2666666666666706
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.5546666666666666,
       "cc": 36.90290826999303,
       "cc_by_type": {
        "score": 36.90290826999303
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.90290826999303
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -11.75076322631999,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": -12.252289678959972
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 7.659686301813146,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -13.597890714988448
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 7.659686301813146
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 68.11810035858215,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 100.0,
        "score": 69.39495148225052
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 68.11810035858215
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -25.53781816870939,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -6.438015909636929
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 28.50650905063489,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 47.838367731614795
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 28.50650905063489
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -0.967848961333846,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -33.333333333333336,
        "score": -6.6771317896619165
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 29.47687169859932,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 32.1280940873946
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 29.47687169859932
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -8.77246629290122,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100.0,
        "score": 25.947409483665673
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 11.06914599494948,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60.0,
        "score": 34.87410465151511
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.06914599494948
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 26.706378828218817,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 33.33333333333333,
        "score": 36.721563322629336
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 26.706378828218817
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 12.856796666407638,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -55.55555555555556,
        "score": 5.753852531522674
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 12.856796666407638
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -11.06811721251573,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100.0,
        "score": -0.9462871214181545
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 25.34251136401156,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 30.57647842984659
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 25.34251136401156
      },
      "model_routing": {
       "accuracy": 0.8202764976958525,
       "cc": 63.2648273073167,
       "cc_by_type": {
        "choice": 82.46753246753246,
        "noul": 63.63636363636365,
        "score": 43.690585818054
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 63.2648273073167
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 7.461255389755934,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 56.93271050707971
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 7.461255389755934
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 50.77332873916746,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 52.4128367281802
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.77332873916746
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 17.749833188153378,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 32.80614981076555
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 17.749833188153378
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 30.59909361582899,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0.0,
        "score": 49.768295340240584
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 30.59909361582899
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 77.04547999886454,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 81.13643999659361
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.04547999886454
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.50079697136124,
      89.18191902573092
     ],
     "I": [
      32.20568334756309,
      41.13745884594617
     ],
     "capability": [
      58.46454287020759,
      64.17133593511335
     ],
     "composite": {
      "A": [
       7.169224741773472,
       12.715245634836354
      ],
      "B": [
       6.623658641306371,
       12.218997228813015
      ],
      "C": [
       4.978628292898244,
       8.830031690858576
      ]
     },
     "seed": 32
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.10935553643324,
     "capability": 62.42104465724772,
     "intelligence": 36.7327337780622
    },
    "per_type": {
     "choice": {
      "accuracy": 0.72,
      "calibration": 88.80529020086236,
      "cc": 63.83398074972732,
      "cc_by_type": {
       "choice": 63.83398074972732
      },
      "cc_tiered": 64.92910863972062,
      "cc_tiered_open": 67.89294454901062,
      "cc_tiered_sealed": 61.96527273043063,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.83398074972732
     },
     "noul": {
      "accuracy": 0.49866666666666665,
      "calibration": 83.63275246769038,
      "cc": -0.2666666666666706,
      "cc_by_type": {
       "noul": -0.2666666666666706
      },
      "cc_tiered": 6.8160800552104925,
      "cc_tiered_open": 13.632160110420985,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.5546666666666666,
      "calibration": 91.89002394074701,
      "cc": 36.90290826999303,
      "cc_by_type": {
       "score": 36.90290826999303
      },
      "cc_tiered": 38.453012639255476,
      "cc_tiered_open": 42.19463838367629,
      "cc_tiered_sealed": 34.71138689483466,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.90290826999303
     }
    },
    "raw_on_A": {
     "calibration": 83.37739283046191,
     "capability": 60.48742810868637,
     "intelligence": 37.59746338691083
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 8.4913980941516,
    "Emphasis on Accuracy 60:20:20": 8.073831983280321,
    "Emphasis on Cost 20:20:60": 7.4560563436657805,
    "Emphasis on Speed 20:60:20": 10.490688528619959,
    "Intelligence only": 7.519195208738335
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Macrocosmos (paid fast-lane order d831408e, PUBLIC visibility)",
   "axes": {
    "calibration": 84.8346261602328,
    "cost": 62.97034212213745,
    "intelligence": 29.414053145614563,
    "speed": 87.14949384308518
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06930761904761903,
      "ece_score": 86.13847619047618,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.13847619047618,
      "score_before_time_limit": 86.13847619047618,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4,
      "brier": 0.1728691851466666,
      "ece": 0.07378479999999997,
      "n": 375,
      "score": 85.24304000000001,
      "score_before_time_limit": 85.24304000000001,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.944,
      "interval90_width": 0.5526666666666668,
      "n": 375,
      "nrps": 0.12630688752888886,
      "score": 83.12236229022221,
      "score_before_time_limit": 83.12236229022221,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10562293333333339
     }
    },
    "score": 84.8346261602328
   },
   "capability": 57.12433965292368,
   "composite_ci95": {
    "A": [
     11.609061681354033,
     25.023635418934923
    ],
    "B": [
     9.629448416551392,
     21.79902883825884
    ],
    "C": [
     8.061848389829189,
     17.377524596482576
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7): base Qwen/Qwen3.5-4B-Base priced as Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the server usage block: 1,094,798 input on 1,500 answered items, 0 output tokens (one forward pass, joint-head readout, no generation). COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 849,038 input / 0 output tokens. All-item usage 1,094,798 input / 0 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.021895960000000003
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.017152282828282833
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "iota-decide-4b (Macrocosmos, Clef joint-head 4B, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4-80GB (FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.141/h). Container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only weights, all 16 weight files sha256-verified; driven one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 36.85502187502523,
    "I_sealed": 21.973084416203896,
    "base": 29.414053145614563,
    "excess": 5.042410688465623,
    "gap": 14.881937458821334,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 59.050171703581924,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 58.47750865051904,
       "judge": 64.40342781806197,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 14.56866804692892,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": 21.739130434782616,
       "judge": 0.0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 36.946225874564846,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.07090909090908,
       "hard": 34.7399074074074,
       "judge": 42.41847457627117,
       "standard": 28.08814814814813
      }
     },
     "sealed|choice": {
      "cc": 47.967773170565664,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 42.75471261709262,
       "judge": 49.75514201762979,
       "standard": 45.89748998101666
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -16.36363636363637,
       "judge": -32.30769230769231,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 17.95148007804603,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 30.493157894736854,
       "hard": 16.664225352112673,
       "judge": 10.76348958333333,
       "standard": 25.03713636363637
      }
     }
    }
   },
   "jevbench_score": 18.926688782419063,
   "key": "iota-decide-4b",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (HF metadata, LICENSE, NOTICE; README notes CC-BY-SA NLI training records), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.38",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "macrocosm-os/iota-decide-4b@bd88855a7e3379468537e0142d5f0ada079aca41",
   "noul_decisive": {
    "acc_among_decisive": 0.84,
    "decisive": 225,
    "decisive_rate": 0.6,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.8346261602328,
     "cost": 62.97034212213745,
     "intelligence": 29.414053145614563,
     "speed": 87.4670144658746
    },
    "board_release": "v1.6.43",
    "capability": 57.12433965292368,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 70,
    "capability_rank": 96,
    "composite_ci95": {
     "A": [
      11.615032461140338,
      25.03863609150094
     ],
     "B": [
      9.632718447097313,
      21.808128076335468
     ],
     "C": [
      8.065994764680791,
      17.387941730208983
     ]
    },
    "jevbench_score": 18.937474020302744,
    "measured_in": "v1.6.38",
    "method": "v1.6",
    "open_board_rank": 100,
    "rank": 135,
    "ranks": {
     "A": 135,
     "B": 145,
     "C": 142,
     "capability": 150
    },
    "scores": {
     "A": 18.937474020302744,
     "B": 16.15725640426057,
     "C": 13.151023625210243
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3842653010000504,
     "p50_s_raw": 0.11713265050002519,
     "p95_s_adjusted": 0.46630211340004735,
     "p95_s_raw": 0.1581510567000237
    },
    "views": {
     "Balanced 33:33:33": 16.933803631615454,
     "Emphasis on Accuracy 60:20:20": 13.382059401682262,
     "Emphasis on Cost 20:20:60": 18.591825785082126,
     "Emphasis on Speed 20:60:20": 20.556488297647732,
     "Intelligence only": 10.17945694092124
    }
   },
   "rank": 153,
   "ranked": true,
   "ranks": {
    "A": 153,
    "B": 163,
    "C": 160,
    "capability": 167
   },
   "repo": "https://huggingface.co/macrocosm-os/iota-decide-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.926688782419063,
    "B": 16.150974528054636,
    "C": 13.143533876679907
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3936773030001405,
    "p50_s_raw": 0.12183865150007023,
    "p95_s_adjusted": 0.48967760110003833,
    "p95_s_raw": 0.16983880055001918
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "First-party host-authored HTTP /v1/systemone bridge (sha256 7c4c9c68a7bdc057) around the repo's documented load_release_model()/systemone() and its vendored joint_schema_model.py (Apache-2.0, from Cloudflare/clef); pre-measurement source review PASS and independent bridge review PASS; no trust_remote_code, no pickle weights. FIRST MEASUREMENT (paid fast-lane order d831408e). CONTEXT: states longer than 16,384 tokens are truncated by the model to 16,384 tokens and answered (15 items on this draw; scored as answered, as for earlier provider-truncated rows). BRIDGE: served through a first-party host-authored bridge around the author's documented entry points, not an author server. REPO: published as macrocosm-os/iota-sys1-4b, renamed to macrocosm-os/iota-decide-4b; measured revision bd88855a (weights/code/config identical at HEAD 745ccae4). PROVENANCE, author statements (unverified): LoRA fine-tune of Qwen/Qwen3.5-4B-Base distilled on Cloudflare Clef teacher probabilities (541,071 records: generated/template decision states plus Open-Jev, typed-decisions, civil_comments, GoEmotions, prompt-injections and 8,932 MNLI/SNLI/BoolQ records under CC-BY-SA); the README states no benchmark data was used and overlapping records were dropped. HARDWARE: A100 80 GB; latency reflects that card. Measured accuracy on the 15 shortened states: 2 of 15 correct (13.3 %), versus 822 of 1485 (55.4 %) on the remaining items; a Score item is correct within half a label step.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 63.25333333333331,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 100.0,
        "score": 39.75999999999995
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.25333333333331
      },
      "calibration_gen": {
       "accuracy": 0.38613861386138615,
       "cc": 4.616781990102507,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -12.0,
        "score": 5.383749999999987
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 4.616781990102507
      },
      "composite_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 19.422861047413896,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": -25.92592592592593,
        "score": 10.210769230769245
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.422861047413896
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 60.32685185185185,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 60.98055555555557
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 60.32685185185185
      },
      "jaggedness": {
       "accuracy": 0.3333333333333333,
       "cc": -12.641671961833303,
       "cc_by_type": {
        "choice": 20.596727622714162,
        "noul": -51.35135135135135,
        "score": -7.170392156862726
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.7028753993610224,
       "cc": 53.352812366788186,
       "cc_by_type": {
        "choice": 61.96178568283877,
        "noul": 50.51546391752577,
        "score": 47.58118750000002
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 53.352812366788186
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -39.16193548387097,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100.0,
        "score": -14.260000000000005
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -19.993658975617734,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -50.0,
        "score": -11.011904761904745
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5745454545454546,
       "cc": 25.22416665263471,
       "cc_by_type": {
        "choice": 57.93935441799805,
        "noul": -16.666666666666664,
        "score": 34.399812206572754
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 25.22416665263471
      },
      "negation_pair_gen": {
       "accuracy": 0.4069767441860465,
       "cc": -10.474999999999989,
       "cc_by_type": {
        "choice": 40.00000000000001,
        "noul": -60.0,
        "score": -11.424999999999974
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.6875,
       "cc": 60.05326231691079,
       "cc_by_type": {
        "choice": 60.05326231691079
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 60.05326231691079
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 60.49063101563102,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 60.00000000000001,
        "score": 51.50192307692309
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 60.49063101563102
      },
      "usecase": {
       "accuracy": 0.5,
       "cc": 26.85113385620917,
       "cc_by_type": {
        "choice": 47.30392156862747,
        "noul": 12.5,
        "score": 20.74948000000003
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 26.85113385620917
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -8.991672771672768,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 0.0,
        "score": -31.736923076923063
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 30.521888888888885,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 24.898999999999994
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 30.521888888888885
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 68.38896113889957,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75.0,
        "score": 46.09608695652175
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.38896113889957
      },
      "el": {
       "accuracy": 0.4,
       "cc": 16.15022346368715,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100.0,
        "score": 98.73
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.15022346368715
      },
      "en": {
       "accuracy": 0.5412684622067767,
       "cc": 24.200852323574026,
       "cc_by_type": {
        "choice": 48.597161372206806,
        "noul": 2.877697841726623,
        "score": 21.127697756788642
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 24.200852323574026
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 69.97578863192912,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 19.715294117647055
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.97578863192912
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -18.429034608378867,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": 10.286666666666678
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 35.50656346749226,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 48.87263157894737
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 35.50656346749226
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 6.236473463687151,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 0.0,
        "score": -31.01125
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 6.236473463687151
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 63.882159373492186,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 19.705714285714304
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.882159373492186
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 15.028362898243758,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": 22.273333333333344
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 15.028362898243758
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -10.676481481481494,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 12.414999999999976
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.53944444444444,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 94.285
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.53944444444444
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -15.770683982683984,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 4.635999999999996
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -39.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5384615384615384,
       "cc": 16.406287599195604,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40.0,
        "score": 19.688604651162777
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 16.406287599195604
      },
      "pt": {
       "accuracy": 0.5357142857142857,
       "cc": 31.94592582736138,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 19.999999999999996,
        "score": 50.08645161290322
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 31.94592582736138
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 12.416418439716315,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 73.845
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 12.416418439716315
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 12.944619565217392,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50.0,
        "score": -45.948750000000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 12.944619565217392
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.72064516129034,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 74.42000000000002
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.72064516129034
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -25.120263653484006,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -91.18000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6173333333333333,
       "cc": 50.57310702462733,
       "cc_by_type": {
        "choice": 50.57310702462733
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.57310702462733
      },
      "noul": {
       "accuracy": 0.504,
       "cc": 0.8000000000000007,
       "cc_by_type": {
        "noul": 0.8000000000000007
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.8000000000000007
      },
      "score": {
       "accuracy": 0.45866666666666667,
       "cc": 23.044314079422456,
       "cc_by_type": {
        "score": 23.044314079422456
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 23.044314079422456
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -4.71392156862746,
       "cc_by_type": {
        "choice": 4.0,
        "noul": -25.0,
        "score": 6.858235294117621
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -19.2541912194086,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -33.333333333333336,
        "score": -31.26153846153843
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 40.176504065040646,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 69.30999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 40.176504065040646
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -8.258944931773884,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 2.123749999999991
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 21.14580314009663,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 25.756250000000016
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 21.14580314009663
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -18.243249475891,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -66.66666666666667,
        "score": -25.170000000000048
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 8.067751203045319,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 25.043589743589735
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 8.067751203045319
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -8.178912286017274,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -50.0,
        "score": 9.08695652173911
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": 18.254074074074072,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 19.999999999999996,
        "score": 38.92888888888889
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.254074074074072
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": 11.594826098241517,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0.0,
        "score": 11.872272727272726
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 11.594826098241517
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 13.114300368021299,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -55.55555555555556,
        "score": 6.526363636363652
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 13.114300368021299
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -47.04889784946236,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100.0,
        "score": -44.372499999999995
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 46.48546918319471,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 60.00000000000001,
        "score": 25.521666666666643
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 46.48546918319471
      },
      "model_routing": {
       "accuracy": 0.7188940092165899,
       "cc": 52.40613658910534,
       "cc_by_type": {
        "choice": 68.44155844155844,
        "noul": 57.57575757575757,
        "score": 31.20109375
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 52.40613658910534
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -11.69606676451771,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 10.977058823529406
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 41.228152393876634,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 23.777307692307716
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 41.228152393876634
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.454253839466833,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": -3.0805882352940905
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.454253839466833
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 11.62452173913043,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 40.670666666666655
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 11.62452173913043
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 70.2200530686146,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50.0,
        "score": 71.33631578947369
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 70.2200530686146
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.65934328285569,
      85.84677132978175
     ],
     "I": [
      24.25545772976716,
      32.98874141715469
     ],
     "capability": [
      52.41656740849436,
      58.60148516850051
     ],
     "composite": {
      "A": [
       11.609061681354033,
       25.023635418934923
      ],
      "B": [
       9.629448416551392,
       21.79902883825884
      ],
      "C": [
       8.061848389829189,
       17.377524596482576
      ]
     },
     "seed": 34
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.8346261602328,
     "capability": 57.12433965292368,
     "intelligence": 29.414053145614563
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6173333333333333,
      "calibration": 86.13847619047618,
      "cc": 50.57310702462733,
      "cc_by_type": {
       "choice": 50.57310702462733
      },
      "cc_tiered": 53.508972437073794,
      "cc_tiered_open": 59.050171703581924,
      "cc_tiered_sealed": 47.967773170565664,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.57310702462733
     },
     "noul": {
      "accuracy": 0.504,
      "calibration": 85.24304000000001,
      "cc": 0.8000000000000007,
      "cc_by_type": {
       "noul": 0.8000000000000007
      },
      "cc_tiered": 7.28433402346446,
      "cc_tiered_open": 14.56866804692892,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.8000000000000007
     },
     "score": {
      "accuracy": 0.45866666666666667,
      "calibration": 83.12236229022221,
      "cc": 23.044314079422456,
      "cc_by_type": {
       "score": 23.044314079422456
      },
      "cc_tiered": 27.44885297630544,
      "cc_tiered_open": 36.946225874564846,
      "cc_tiered_sealed": 17.95148007804603,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.044314079422456
     }
    },
    "raw_on_A": {
     "calibration": 81.68506181215012,
     "capability": 55.999809766504924,
     "intelligence": 30.314557720859717
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.92230659663555,
    "Emphasis on Accuracy 60:20:20": 13.377749880202591,
    "Emphasis on Cost 20:20:60": 18.583508668668784,
    "Emphasis on Speed 20:60:20": 20.526016513830232,
    "Intelligence only": 10.17945694092124
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "szybkie-ai",
   "axes": {
    "calibration": 89.98387597336676,
    "cost": 42.92470023353102,
    "intelligence": 33.22856158102424,
    "speed": 92.86051128678476
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.051101311475409826,
      "ece_score": 89.77973770491803,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.77973770491803,
      "score_before_time_limit": 89.77973770491803,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4584450402144772,
      "brier": 0.1341961465415548,
      "ece": 0.047425201072386014,
      "n": 373,
      "score": 90.51495978552279,
      "score_before_time_limit": 90.51495978552279,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.5580197132616489,
      "n": 372,
      "nrps": 0.10785977850358425,
      "score": 89.6569304296595,
      "score_before_time_limit": 89.6569304296595,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0495008064516129
     }
    },
    "score": 89.98387597336676
   },
   "capability": 61.6062187771955,
   "composite_ci95": {
    "A": [
     11.410249423655436,
     23.242201975658205
    ],
    "B": [
     9.924999326294781,
     21.248287672490555
    ],
    "C": [
     7.923784321982943,
     16.14041803865153
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.6-35B-A3B -> qwen/qwen3.6-35b-a3b USD 0.15/M input, USD 1.0/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the author's server, KEPT AS SENT: output_tokens is 1 per answered question (the label position the engine counts, not generated text); this adds about USD 0.0002 per 1,000 decisions at the reference output rate. OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.079893 per 1,000 decisions on the common basis (USD 0.079094 over all 1,500 items) is 1.237x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.079893 per 1,000 decisions (1.237x the official cost cap USD 0.064595), against USD 0.079094 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0790943
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.07989323232323232
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JEB-35B-A3B (szybkie.ai, Qwen3.6-35B-A3B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's JEB server over vLLM (raw label probabilities, no calibration file, one presentation per question).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 38.44350210899075,
    "I_sealed": 28.013621053057726,
    "base": 33.22856158102424,
    "excess": 0.5903542855773125,
    "gap": 10.429881055933024,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 68.35657646052876,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 52.537903757415954,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 12.366459627329192,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 39.13043478260869,
       "judge": -30.000000000000004,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 34.607470239114306,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.647878787878795,
       "hard": 27.61083333333334,
       "judge": 50.22610169491526,
       "standard": 18.152592592592566
      }
     },
     "sealed|choice": {
      "cc": 56.57345439924718,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 44.836359431016525,
       "judge": 58.570029382957905,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 20.68965517241379,
       "hard": -10.909090909090914,
       "judge": -32.30769230769231,
       "standard": 19.402985074626855
      }
     },
     "sealed|score": {
      "cc": 27.467408759926002,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 49.020087719298246,
       "hard": 21.118253521126817,
       "judge": 25.38817708333333,
       "standard": 32.50822727272726
      }
     }
    }
   },
   "jevbench_score": 17.29726912937929,
   "key": "jeb-35b-a3b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "szybkie-ai/jeb-35b-a3b@dfac7afaed21ffff638ae4588402db560376aeac",
   "noul_decisive": {
    "acc_among_decisive": 0.9356435643564357,
    "decisive": 202,
    "decisive_rate": 0.5415549597855228,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.98387597336676,
     "cost": 42.92470023353102,
     "intelligence": 33.22856158102424,
     "speed": 92.99325391268877
    },
    "board_release": "v1.6.43",
    "capability": 61.6062187771955,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      11.412397610637285,
      23.247151641339965
     ],
     "B": [
      9.926306265903968,
      21.251604442413498
     ],
     "C": [
      7.925276118498115,
      16.143855306486085
     ]
    },
    "jevbench_score": 17.300802214285724,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 109,
    "rank": 145,
    "ranks": {
     "A": 145,
     "B": 147,
     "C": 148,
     "capability": 127
    },
    "scores": {
     "A": 17.300802214285724,
     "B": 15.448393624283149,
     "C": 12.014445982142869
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.22059132121503353,
     "p50_s_raw": 0.035295660607516766,
     "p95_s_adjusted": 0.22755489745177326,
     "p95_s_raw": 0.03877744872588664
    },
    "views": {
     "Balanced 33:33:33": 15.22363233434469,
     "Emphasis on Accuracy 60:20:20": 13.089956907787629,
     "Emphasis on Cost 20:20:60": 14.69709047732901,
     "Emphasis on Speed 20:60:20": 19.00168433057001,
     "Intelligence only": 10.816061538150686
    }
   },
   "rank": 164,
   "ranked": true,
   "ranks": {
    "A": 164,
    "B": 165,
    "C": 168,
    "capability": 143
   },
   "repo": "https://huggingface.co/szybkie-ai/jeb-35b-a3b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.29726912937929,
    "B": 15.446139885553869,
    "C": 12.011992450957845
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2214831450022757,
    "p50_s_raw": 0.03574157250113785,
    "p95_s_adjusted": 0.2336728145368397,
    "p95_s_raw": 0.041836407268419855
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 59.72666666666665,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 4.179999999999939
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 59.72666666666665
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 11.705334371686106,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -4.0000000000000036,
        "score": 5.924062499999994
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 11.705334371686106
      },
      "composite_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 19.52070356472796,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -33.333333333333336,
        "score": 24.415769230769236
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.52070356472796
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 84.24076719576719,
       "cc_by_type": {
        "choice": 90.47619047619048,
        "noul": 100.0,
        "score": 62.24611111111111
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.24076719576719
      },
      "jaggedness": {
       "accuracy": 0.4222222222222222,
       "cc": 0.657932106935687,
       "cc_by_type": {
        "choice": 30.702598652550538,
        "noul": -51.35135135135135,
        "score": 22.622549019607874
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.657932106935687
      },
      "llm_router": {
       "accuracy": 0.7412140575079872,
       "cc": 56.16464470490973,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 48.453608247422686,
        "score": 50.94637499999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.16464470490973
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": -7.445064637538863,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -75.0,
        "score": 14.520476190476206
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6218181818181818,
       "cc": 30.28596949640944,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": -13.888888888888884,
        "score": 38.98220657276996
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.28596949640944
      },
      "negation_pair_gen": {
       "accuracy": 0.5697674418604651,
       "cc": 10.81088888888889,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -46.666666666666664,
        "score": 5.765999999999993
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 10.81088888888889
      },
      "option_order": {
       "accuracy": 0.6666666666666666,
       "cc": 57.3901464713715,
       "cc_by_type": {
        "choice": 57.3901464713715
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 57.3901464713715
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 73.82135096635096,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 80.0,
        "score": 65.48807692307692
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 73.82135096635096
      },
      "usecase": {
       "accuracy": 0.5454545454545454,
       "cc": 21.5768768627451,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": -20.833333333333336,
        "score": 23.554160000000003
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 21.5768768627451
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 7.329047619047614,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -20.87000000000001
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 7.329047619047614
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.5384615384615384,
       "cc": 5.657004096262161,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": -8.22099999999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 5.657004096262161
      },
      "de": {
       "accuracy": 0.75,
       "cc": 64.25974605617546,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 50.0,
        "score": 66.67304347826088
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 64.25974605617546
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 23.614667908131594,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 87.79
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 23.614667908131594
      },
      "en": {
       "accuracy": 0.5977410947002606,
       "cc": 29.215352744871566,
       "cc_by_type": {
        "choice": 57.38397139405179,
        "noul": 2.158273381294973,
        "score": 28.103813459267933
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 29.215352744871566
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 48.755200396635,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 50.0,
        "score": 6.053529411764713
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 48.755200396635
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 18.362003642987244,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 86.77999999999999
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.362003642987244
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 75.0780185758514,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 46.41052631578948
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.0780185758514
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 29.694657821229047,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25.0,
        "score": 30.843749999999982
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 29.694657821229047
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 63.80406413539694,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 19.47142857142856
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.80406413539694
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 17.03858512046597,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0.0,
        "score": 8.30399999999999
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 17.03858512046597
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -5.48076719576719,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 28.002142857142875
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.07833333333332,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 94.90166666666666
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.07833333333332
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 14.425538239538243,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 28.558000000000007
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.425538239538243
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 40.080546900373,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0.0,
        "score": 35.47651162790697
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 40.080546900373
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 42.31917558499819,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 19.999999999999996,
        "score": 38.77838709677419
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 42.31917558499819
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 13.50039007092198,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -60.0,
        "score": 51.565
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 13.50039007092198
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 24.365416666666665,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": -26.903750000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 24.365416666666665
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.10397849462366,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 63.570000000000014
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.10397849462366
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 18.364312617702463,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": 13.85000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.364312617702463
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6866666666666666,
       "cc": 59.528502267552,
       "cc_by_type": {
        "choice": 59.528502267552
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.528502267552
      },
      "noul": {
       "accuracy": 0.504,
       "cc": 0.8000000000000007,
       "cc_by_type": {
        "noul": 0.8000000000000007
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.8000000000000007
      },
      "score": {
       "accuracy": 0.552,
       "cc": 29.92095667870036,
       "cc_by_type": {
        "score": 29.92095667870036
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 29.92095667870036
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -23.629607843137265,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": -47.888823529411795
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": -0.203754047884478,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": -11.37730769230767
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 31.193170731707323,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 42.36000000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 31.193170731707323
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -30.005031676413257,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -8.143750000000004
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 10.044051932367147,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 6.9437499999999925
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 10.044051932367147
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -24.42324947589097,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -66.66666666666667,
        "score": -43.70999999999996
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 9.24208503914385,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": 38.37051282051279
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 9.24208503914385
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": -9.992762207746297,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100.0,
        "score": 22.28652173913044
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 14.67074074074074,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 66.51222222222222
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 14.67074074074074
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 33.534732550342824,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 18.662727272727274
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.534732550342824
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 34.76682562054655,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 4.817272727272737
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 34.76682562054655
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -49.31033602150538,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -18.898750000000007
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 7.846555591099727,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -60.0,
        "score": 18.088611111111085
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 7.846555591099727
      },
      "model_routing": {
       "accuracy": 0.7557603686635944,
       "cc": 56.35260518127705,
       "cc_by_type": {
        "choice": 76.62337662337664,
        "noul": 54.54545454545454,
        "score": 37.888984374999964
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 56.35260518127705
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 14.596378382447028,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 66.82176470588234
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 14.596378382447028
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 53.74200354736566,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 44.60576923076924
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 53.74200354736566
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 16.266214623780545,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 28.355294117647045
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 16.266214623780545
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -7.030033816425129,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -66.66666666666667,
        "score": 18.040333333333315
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 56.419999999999995,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 69.26
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 56.419999999999995
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.93418214562868,
      89.85283877896852
     ],
     "I": [
      28.037739089154297,
      37.67898728636531
     ],
     "capability": [
      56.857189752779114,
      62.91121122015222
     ],
     "composite": {
      "A": [
       11.410249423655436,
       23.242201975658205
      ],
      "B": [
       9.924999326294781,
       21.248287672490555
      ],
      "C": [
       7.923784321982943,
       16.14041803865153
      ]
     },
     "seed": 33
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.98387597336676,
     "capability": 61.6062187771955,
     "intelligence": 33.22856158102424
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 89.77973770491803,
      "cc": 59.528502267552,
      "cc_by_type": {
       "choice": 59.528502267552
      },
      "cc_tiered": 62.46501542988797,
      "cc_tiered_open": 68.35657646052876,
      "cc_tiered_sealed": 56.57345439924718,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.528502267552
     },
     "noul": {
      "accuracy": 0.504,
      "calibration": 90.51495978552279,
      "cc": 0.8000000000000007,
      "cc_by_type": {
       "noul": 0.8000000000000007
      },
      "cc_tiered": 6.183229813664596,
      "cc_tiered_open": 12.366459627329192,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.8000000000000007
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 89.6569304296595,
      "cc": 29.92095667870036,
      "cc_by_type": {
       "score": 29.92095667870036
      },
      "cc_tiered": 31.037439499520154,
      "cc_tiered_open": 34.607470239114306,
      "cc_tiered_sealed": 27.467408759926002,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.92095667870036
     }
    },
    "raw_on_A": {
     "calibration": 86.05549626571135,
     "capability": 59.931312510984895,
     "intelligence": 33.807128756258436
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.219984949939533,
    "Emphasis on Accuracy 60:20:20": 13.08833874197717,
    "Emphasis on Cost 20:20:60": 14.695050606199946,
    "Emphasis on Speed 20:60:20": 18.991459126966355,
    "Intelligence only": 10.816061538150686
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "frontier-infra",
   "axes": {
    "calibration": 83.31146464049283,
    "cost": 48.85607041331758,
    "intelligence": 30.5871791271758,
    "speed": 91.458678539542
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11935933114754105,
      "ece_score": 76.12813377049179,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 76.12813377049179,
      "score_before_time_limit": 76.12813377049179,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.48257372654155495,
      "brier": 0.14617931767116624,
      "ece": 0.06136039410187674,
      "n": 373,
      "score": 87.72792117962464,
      "score_before_time_limit": 87.72792117962464,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9865591397849462,
      "interval90_width": 0.6666666666666664,
      "n": 372,
      "nrps": 0.10575929584157615,
      "score": 86.07833897136206,
      "score_before_time_limit": 86.07833897136206,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.08633696236559144
     }
    },
    "score": 83.31146464049283
   },
   "capability": 56.949321883834315,
   "composite_ci95": {
    "A": [
     14.318906654648984,
     25.032506646342735
    ],
    "B": [
     12.28540134381899,
     22.393829566503097
    ],
    "C": [
     9.94368517683957,
     17.383685171071352
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-9B -> qwen/qwen3.5-9b USD 0.09999999999999999/M input, USD 0.15/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the author's server, KEPT AS SENT: output_tokens is 1 per answered question (the label position the engine counts, not generated text); this adds about USD 0.0002 per 1,000 decisions at the reference output rate. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.050675 per 1,000 decisions (0.785x the official cost cap USD 0.064595), against USD 0.050169 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05016869999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05067545454545454
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jebadiah 9B v2 (Frontier Infra, Qwen3.5-9B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's standalone jebadiah_server: bf16, label-token logits, strict prompt contract, fitted per-type temperatures, fp32 candidate logits.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 34.34712859027669,
    "I_sealed": 26.827229664074913,
    "base": 30.5871791271758,
    "excess": -2.319627844153935,
    "gap": 7.519898926201776,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 63.94109602078727,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": -30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 39.1002897500428,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.8623575757576,
       "hard": 38.462709259259256,
       "judge": 35.66567627118642,
       "standard": 41.64633703703704
      }
     },
     "sealed|choice": {
      "cc": 53.67345673395262,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 71.41041931385007,
       "hard": 47.4384179484214,
       "judge": 52.39960822722822,
       "standard": 59.185825775152935
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 31.034482758620683,
       "hard": -12.727272727272732,
       "judge": -47.692307692307686,
       "standard": 7.462686567164178
      }
     },
     "sealed|score": {
      "cc": 26.808232258272128,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 43.64942105263159,
       "hard": 18.956308450704242,
       "judge": 24.474275000000013,
       "standard": 37.59242136363634
      }
     }
    }
   },
   "jevbench_score": 18.78097985205501,
   "key": "jebadiah-9b-v2",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "frontier-infra/jebadiah-9b-v2@bc63609af6434fdf18ed8b89e19df96db5a24944",
   "noul_decisive": {
    "acc_among_decisive": 0.9326424870466321,
    "decisive": 193,
    "decisive_rate": 0.517426273458445,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.31146464049283,
     "cost": 48.85607041331758,
     "intelligence": 30.5871791271758,
     "speed": 91.57148361254983
    },
    "board_release": "v1.6.43",
    "capability": 56.949321883834315,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 72,
    "capability_rank": 98,
    "composite_ci95": {
     "A": [
      14.321319372467606,
      25.03713540072477
     ],
     "B": [
      12.286822142404677,
      22.39679324818056
     ],
     "C": [
      9.945360675324725,
      17.38689958383665
     ]
    },
    "jevbench_score": 18.78430461690739,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 102,
    "rank": 137,
    "ranks": {
     "A": 137,
     "B": 144,
     "C": 143,
     "capability": 152
    },
    "scores": {
     "A": 18.78430461690739,
     "B": 16.423355223462213,
     "C": 13.044655983963466
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25783845791593196,
     "p50_s_raw": 0.05391922895796597,
     "p95_s_adjusted": 0.27008716338314115,
     "p95_s_raw": 0.06004358169157058
    },
    "views": {
     "Balanced 33:33:33": 16.727077379556174,
     "Emphasis on Accuracy 60:20:20": 13.798758909175735,
     "Emphasis on Cost 20:20:60": 17.01136530433605,
     "Emphasis on Speed 20:60:20": 20.79200680874768,
     "Intelligence only": 10.92887170611703
    }
   },
   "rank": 155,
   "ranked": true,
   "ranks": {
    "A": 155,
    "B": 162,
    "C": 161,
    "capability": 169
   },
   "repo": "https://huggingface.co/frontier-infra/jebadiah-9b-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.78097985205501,
    "B": 16.421321894937,
    "C": 13.042347119482647
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2601211030036211,
    "p50_s_raw": 0.05506055150181055,
    "p95_s_adjusted": 0.2747619305271655,
    "p95_s_raw": 0.06238096526358276
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 61.07781111111111,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100.0,
        "score": 24.90009999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 61.07781111111111
      },
      "calibration_gen": {
       "accuracy": 0.40594059405940597,
       "cc": 10.446769936373283,
       "cc_by_type": {
        "choice": 17.285259809119836,
        "noul": 12.00000000000001,
        "score": 2.0550500000000027
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 10.446769936373283
      },
      "composite_gen": {
       "accuracy": 0.5346534653465347,
       "cc": 18.094449115187732,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -40.74074074074075,
        "score": 24.29238076923077
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 18.094449115187732
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 56.24330185185185,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 48.72990555555556
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 56.24330185185185
      },
      "jaggedness": {
       "accuracy": 0.36666666666666664,
       "cc": -1.3864631875152067,
       "cc_by_type": {
        "choice": 24.927815206929747,
        "noul": -45.945945945945944,
        "score": 16.85874117647058
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.7699680511182109,
       "cc": 62.21615746389665,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 56.70103092783505,
        "score": 59.268564999999995
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.21615746389665
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -20.892842873506787,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100.0,
        "score": 23.91940952380953
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6,
       "cc": 25.841182948220062,
       "cc_by_type": {
        "choice": 63.808281708509945,
        "noul": -25.0,
        "score": 38.71526713615023
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 25.841182948220062
      },
      "negation_pair_gen": {
       "accuracy": 0.4418604651162791,
       "cc": -6.473503333333329,
       "cc_by_type": {
        "choice": 56.66666666666668,
        "noul": -66.66666666666667,
        "score": -9.420509999999993
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 68.60171563101564,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 80.0,
        "score": 55.835176923076936
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 68.60171563101564
      },
      "usecase": {
       "accuracy": 0.4863636363636364,
       "cc": 18.583117720261452,
       "cc_by_type": {
        "choice": 49.75490196078433,
        "noul": -25.0,
        "score": 30.99445120000003
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 18.583117720261452
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 17.72341184371184,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50.0,
        "score": -1.5916692307692326
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 17.72341184371184
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 24.637396492575522,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 27.982849999999992
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 24.637396492575522
      },
      "de": {
       "accuracy": 0.75,
       "cc": 67.01662662562524,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75.0,
        "score": 57.908286956521735
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 67.01662662562524
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 24.128667908131593,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 89.33200000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 24.128667908131593
      },
      "en": {
       "accuracy": 0.5664639443961772,
       "cc": 26.84201593477144,
       "cc_by_type": {
        "choice": 54.747928387498305,
        "noul": -2.1582733812949617,
        "score": 27.936392798110976
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 26.84201593477144
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 67.20150275405432,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 21.180364705882337
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.20150275405432
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 15.402943169398917,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 45.11593333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 15.402943169398917
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 55.9409701754386,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 67.8229105263158
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 55.9409701754386
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 5.854853142458097,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -100.0,
        "score": 51.08411249999999
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 5.854853142458097
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 64.8121273057937,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 8.465999999999962
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 64.8121273057937
      },
      "ja": {
       "accuracy": 0.4,
       "cc": -0.4558023193010274,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": -40.0,
        "score": 24.41496000000002
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -7.808200529100527,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 21.01984285714287
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 95.45897222222222,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 86.37691666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 95.45897222222222
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 1.8647846753246806,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 31.56838000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.8647846753246806
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 30.657935079241657,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -19.999999999999996,
        "score": 34.82611162790696
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 30.657935079241657
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 41.932065296126616,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 19.999999999999996,
        "score": 48.224009677419346
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 41.932065296126616
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -12.919919858156026,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60.0,
        "score": 23.3679
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": -2.1177119565217524,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": -62.87487500000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 66.71357365591398,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 64.65685
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 66.71357365591398
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 14.517088888888878,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -23.11540000000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.517088888888878
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6626666666666666,
       "cc": 56.428557760385765,
       "cc_by_type": {
        "choice": 56.428557760385765
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 56.428557760385765
      },
      "noul": {
       "accuracy": 0.48,
       "cc": -4.0000000000000036,
       "cc_by_type": {
        "noul": -4.0000000000000036
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.504,
       "cc": 30.3285861010831,
       "cc_by_type": {
        "score": 30.3285861010831
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 30.3285861010831
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -20.217698039215694,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": -37.65309411764708
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 6.881704004530096,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -11.111111111111116,
        "score": 6.290384615384637
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 6.881704004530096
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 30.13055826558266,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0.0,
        "score": 22.91200000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.13055826558266
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -35.508502631578956,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": -1.262350000000012
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 18.20034864130435,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 45.90539375
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 18.20034864130435
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -46.83123815513627,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100.0,
        "score": -52.443400000000004
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 12.310430690224806,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 37.771628205128195
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 12.310430690224806
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -14.343936656062217,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 30.13892173913042
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -5.765200000000003,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60.0,
        "score": 46.871066666666664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 9.370813276231262,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0.0,
        "score": 18.0482
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 9.370813276231262
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 35.32868872053872,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -11.111111111111116,
        "score": 17.097177272727283
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 35.32868872053872
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -59.25189018817205,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -48.72341250000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.4074074074074074,
       "cc": 20.06767288689841,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 26.268277777777772
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 20.06767288689841
      },
      "model_routing": {
       "accuracy": 0.8110599078341014,
       "cc": 66.99794162946428,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 72.72727272727273,
        "score": 48.13668203124999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.99794162946428
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 9.025348921756807,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 73.14130588235295
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 9.025348921756807
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 58.44408162393162,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 41.99891153846155
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 58.44408162393162
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 14.167175939341256,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50.0,
        "score": 7.279852941176479
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 14.167175939341256
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 8.994591932367145,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": 37.12870333333333
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 8.994591932367145
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 71.28822061247423,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50.0,
        "score": 74.54081842105262
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 71.28822061247423
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.35230824724442,
      85.74985775934907
     ],
     "I": [
      27.368604889402985,
      34.51967568447579
     ],
     "capability": [
      54.162736400093735,
      58.62101341997018
     ],
     "composite": {
      "A": [
       14.318906654648984,
       25.032506646342735
      ],
      "B": [
       12.28540134381899,
       22.393829566503097
      ],
      "C": [
       9.94368517683957,
       17.383685171071352
      ]
     },
     "seed": 34
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.31146464049283,
     "capability": 56.949321883834315,
     "intelligence": 30.5871791271758
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6626666666666666,
      "calibration": 76.12813377049179,
      "cc": 56.428557760385765,
      "cc_by_type": {
       "choice": 56.428557760385765
      },
      "cc_tiered": 58.807276377369945,
      "cc_tiered_open": 63.94109602078727,
      "cc_tiered_sealed": 53.67345673395262,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.428557760385765
     },
     "noul": {
      "accuracy": 0.48,
      "calibration": 87.72792117962464,
      "cc": -4.0000000000000036,
      "cc_by_type": {
       "noul": -4.0000000000000036
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.504,
      "calibration": 86.07833897136206,
      "cc": 30.3285861010831,
      "cc_by_type": {
       "score": 30.3285861010831
      },
      "cc_tiered": 32.95426100415746,
      "cc_tiered_open": 39.1002897500428,
      "cc_tiered_sealed": 26.808232258272128,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.3285861010831
     }
    },
    "raw_on_A": {
     "calibration": 76.8381791401458,
     "capability": 54.30955267637154,
     "intelligence": 31.780926212597272
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.723562299985588,
    "Emphasis on Accuracy 60:20:20": 13.79732351087812,
    "Emphasis on Cost 20:20:60": 17.009183779322694,
    "Emphasis on Speed 20:60:20": 20.782233369148024,
    "Intelligence only": 10.92887170611703
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Frontier Infra",
   "axes": {
    "calibration": 89.67673072100749,
    "cost": 48.85607041331758,
    "intelligence": 37.15024789587638,
    "speed": 90.97576909567059
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0545485704918033,
      "ece_score": 89.09028590163935,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.09028590163935,
      "score_before_time_limit": 89.09028590163935,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4048257372654156,
      "brier": 0.12966919167437543,
      "ece": 0.02817181233243971,
      "n": 373,
      "score": 94.36563753351206,
      "score_before_time_limit": 94.36563753351206,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9408602150537635,
      "interval90_width": 0.4444444444444445,
      "n": 372,
      "nrps": 0.10121750178666497,
      "score": 85.57426872787106,
      "score_before_time_limit": 85.57426872787106,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09364856182795694
     }
    },
    "score": 89.67673072100749
   },
   "capability": 63.41348930844194,
   "composite_ci95": {
    "A": [
     21.576286920084637,
     38.2033180855225
    ],
    "B": [
     19.014124679975207,
     35.0261728885226
    ],
    "C": [
     14.983532583392108,
     26.530082003835073
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-9B -> qwen/qwen3.5-9b USD 0.09999999999999999/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 533d80f0...d960e5, the module this row was first priced with (the later re-freezes only add hosted-API launch-list keys), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). Tokens as reported by the author's own server (750,303 input over the 1,485 answered items, output_tokens 1 per answered request). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.050675 per 1,000 decisions (0.785x the official cost cap USD 0.064595), against USD 0.050169 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05016869999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05067545454545454
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jebadiah 9B v2.1 (Frontier Infra, Qwen3.5-9B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA GeForce RTX 5090 32 GB (sm_120, driver 595.71.05), Romania (EU) - different hardware from the RTX PRO 6000 the author measures on, so Speed is a lower bound on this card and not comparable to their 49.9 ms median. The author's OWN standalone server `jebadiah-serve --model <pinned local dir> --device cuda` from getainode/jebadiah @ 74a0cd92965529e90a411532e9887b544c931b2d (the commit behind the published jebadiah-27b row, independently code-reviewed PASS), installed from the author's own frozen uv.lock (`uv sync --frozen --no-dev --extra cuda`, torch 2.14.0+cu130, transformers 5.17.0, flash-linear-attention 0.5.2), all author defaults (dtype auto = bf16, batch-size 8, --max-prompt-tokens 8192, strict prompt-contract check ON, the repo's own temperatures.json, fp32 candidate logits), JEBADIAH_API_KEY unset. Isolation: server started inside a network namespace with loopback only (unshare -n), the cut proven before the sealed input was staged and re-proven immediately before the first sealed request, synthetic smoke gate passed in between; runner unchanged (run_v15.py, 0 resumes, rc 0); one use of input-selfhosted-1500 (custody #20323, deputy #20315). Pod removed by exact id and verified gone four ways. Published Speed uses the standing x2 + 0.15 s self-hosted adjustment; raw p50 0.0502 s / p95 0.0837 s.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 45.02812038711811,
    "I_sealed": 29.272375404634655,
    "base": 37.15024789587638,
    "excess": 5.916218212127742,
    "gap": 15.755744982483453,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 65.08523959814883,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 60.36489462094998,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 22.648033126294003,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 47.35108843691149,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.52056666666667,
       "hard": 53.42431944444446,
       "judge": 36.94557627118644,
       "standard": 49.22815555555554
      }
     },
     "sealed|choice": {
      "cc": 56.24785832875334,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 69.50444726810673,
       "hard": 49.52006476234531,
       "judge": 55.04407443682666,
       "standard": 64.88082682978275
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 37.93103448275863,
       "hard": -3.6363636363636376,
       "judge": -26.15384615384615,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 31.56926788515063,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 54.8699552631579,
       "hard": 24.433701408450716,
       "judge": 25.61958750000002,
       "standard": 43.11457772727274
      }
     }
    }
   },
   "jevbench_score": 30.32311466490751,
   "key": "jebadiah-9b-v2-1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0 (weights and base)",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "frontier-infra/jebadiah-9b-v2-1@c5e49fab7ec8e9923e1f907d85911a53fb8e9c32",
   "noul_decisive": {
    "acc_among_decisive": 0.9279279279279279,
    "decisive": 222,
    "decisive_rate": 0.5951742627345844,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.67673072100749,
     "cost": 48.85607041331758,
     "intelligence": 37.15024789587638,
     "speed": 91.39972504887733
    },
    "board_release": "v1.6.43",
    "capability": 63.41348930844194,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 44,
    "capability_rank": 68,
    "composite_ci95": {
     "A": [
      21.591093369152567,
      38.23238527743113
     ],
     "B": [
      19.023355726025706,
      35.04565170961342
     ],
     "C": [
      14.993814839689282,
      26.55026755377161
     ]
    },
    "jevbench_score": 30.34536701293529,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": 68,
    "rank": 100,
    "ranks": {
     "A": 100,
     "B": 101,
     "C": 105,
     "capability": 110
    },
    "scores": {
     "A": 30.34536701293529,
     "B": 27.339601213975246,
     "C": 21.073171536760626
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2362852499994915,
     "p50_s_raw": 0.04314262499974575,
     "p95_s_adjusted": 0.30661322509549793,
     "p95_s_raw": 0.07830661254774895
    },
    "views": {
     "Balanced 33:33:33": 27.110180335527737,
     "Emphasis on Accuracy 60:20:20": 23.49650090011741,
     "Emphasis on Cost 20:20:60": 26.54977055133271,
     "Emphasis on Speed 20:60:20": 32.85701676126051,
     "Intelligence only": 19.581330456126626
    }
   },
   "rank": 114,
   "ranked": true,
   "ranks": {
    "A": 114,
    "B": 115,
    "C": 120,
    "capability": 128
   },
   "repo": "https://huggingface.co/frontier-infra/jebadiah-9b-v2-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 30.32311466490751,
    "B": 27.325148336139474,
    "C": 21.05771851729689
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25061776800284863,
    "p50_s_raw": 0.0503088840014243,
    "p95_s_adjusted": 0.3187214060009864,
    "p95_s_raw": 0.08436070300049323
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 45.461111111111116,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100.0,
        "score": -21.94999999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 45.461111111111116
      },
      "calibration_gen": {
       "accuracy": 0.39603960396039606,
       "cc": 8.965266960498408,
       "cc_by_type": {
        "choice": 23.647932131495228,
        "noul": -4.0000000000000036,
        "score": 7.247868749999997
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 8.965266960498408
      },
      "composite_gen": {
       "accuracy": 0.5742574257425742,
       "cc": 25.502561765455255,
       "cc_by_type": {
        "choice": 64.22764227642277,
        "noul": -18.518518518518523,
        "score": 30.79856153846151
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 25.502561765455255
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 79.02633518518519,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 77.07900555555555
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 79.02633518518519
      },
      "jaggedness": {
       "accuracy": 0.42777777777777776,
       "cc": 7.105247243920597,
       "cc_by_type": {
        "choice": 32.14629451395574,
        "noul": -35.13513513513513,
        "score": 24.304582352941186
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 7.105247243920597
      },
      "llm_router": {
       "accuracy": 0.7955271565495208,
       "cc": 68.52774777747052,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 71.13402061855669,
        "score": 63.770346249999996
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.52774777747052
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -21.368183456062837,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -100.0,
        "score": -14.620014285714289
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6145454545454545,
       "cc": 30.838639188636876,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": -16.666666666666664,
        "score": 43.41799342723004
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.838639188636876
      },
      "negation_pair_gen": {
       "accuracy": 0.5116279069767442,
       "cc": 10.267682222222229,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": -33.333333333333336,
        "score": 4.136380000000028
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 10.267682222222229
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 70.84381691306692,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 80.0,
        "score": 62.56148076923078
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.84381691306692
      },
      "usecase": {
       "accuracy": 0.55,
       "cc": 27.858113095424844,
       "cc_by_type": {
        "choice": 52.205882352941195,
        "noul": -4.1666666666666625,
        "score": 35.535123600000006
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 27.858113095424844
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 4.7372142857142805,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -28.645500000000013
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 4.7372142857142805
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 39.28832631848439,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 26.006300000000017
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 39.28832631848439
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 70.35387155316147,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75.0,
        "score": 67.92002173913043
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 70.35387155316147
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 21.789848975791433,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -100.0,
        "score": 98.8891
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.789848975791433
      },
      "en": {
       "accuracy": 0.6072980017376195,
       "cc": 35.34340890177539,
       "cc_by_type": {
        "choice": 56.28562014132117,
        "noul": 14.388489208633093,
        "score": 35.35611735537191
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 35.34340890177539
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.55888314621119,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 19.252505882352942
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.55888314621119
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 29.99688269581056,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 56.110866666666666
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 29.99688269581056
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 55.07443787409701,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 86.39978421052632
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 55.07443787409701
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 3.0043369878957136,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50.0,
        "score": -24.227212500000018
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 3.0043369878957136
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 61.87361175444456,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 13.680071428571395
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 61.87361175444456
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 25.42277845379931,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0.0,
        "score": 33.45658
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 25.42277845379931
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -7.526621957671959,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 21.864578571428574
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.65298333333334,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 96.62561666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.65298333333334
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -0.26805731601731964,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 51.14387999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 32.30207713293114,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 32.141102325581386
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 32.30207713293114
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 45.18160400580404,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 19.999999999999996,
        "score": 57.97262580645162
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 45.18160400580404
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -0.5227982269503476,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 35.02735000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 10.888600543478256,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50.0,
        "score": -73.85593750000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 10.888600543478256
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.77072365591397,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 73.82829999999998
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.77072365591397
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 13.044855555555557,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -27.532099999999993
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 13.044855555555557
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6813333333333333,
       "cc": 58.83962571040395,
       "cc_by_type": {
        "choice": 58.83962571040395
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 58.83962571040395
      },
      "noul": {
       "accuracy": 0.5493333333333333,
       "cc": 9.866666666666667,
       "cc_by_type": {
        "noul": 9.866666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 9.866666666666667
      },
      "score": {
       "accuracy": 0.544,
       "cc": 36.35312382671484,
       "cc_by_type": {
        "score": 36.35312382671484
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.35312382671484
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -11.250268627450984,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -50.0,
        "score": -51.75080588235295
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 16.864947946417512,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 11.111111111111116,
        "score": -4.6156461538461535
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 16.864947946417512
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 36.58365826558265,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0.0,
        "score": 42.27129999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 36.58365826558265
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -25.61673798732944,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100.0,
        "score": 16.717037499999996
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 16.98094864130434,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 42.24719374999998
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 16.98094864130434
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -50.77358270440254,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -100.0,
        "score": -76.84905000000009
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 14.320775486604896,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -71.42857142857143,
        "score": 33.998741025641024
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 14.320775486604896
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -9.09330015654193,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100.0,
        "score": 35.437869565217404
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -0.8783444444444442,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60.0,
        "score": 40.698299999999996
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 33.657132663898075,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 33.33333333333333,
        "score": 44.72585909090909
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.657132663898075
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 32.8623636676846,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 21.326109090909096
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 32.8623636676846
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -54.82240618279571,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100.0,
        "score": -67.69302500000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 22.22430251652804,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 32.73816666666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 22.22430251652804
      },
      "model_routing": {
       "accuracy": 0.8248847926267281,
       "cc": 69.68780904581529,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 75.75757575757575,
        "score": 53.17598124999998
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.68780904581529
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 15.739794068721551,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 70.25201176470588
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 15.739794068721551
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 56.08058175249388,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 51.62150384615387
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.08058175249388
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.892261682604083,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": -1.7665647058823453
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.892261682604083
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 3.1867240579710128,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 34.19785333333333
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 3.1867240579710128
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 76.21831885808827,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50.0,
        "score": 89.33111315789473
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 76.21831885808827
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.9065626572593,
      89.19278572343606
     ],
     "I": [
      32.34737884360564,
      41.02255692898279
     ],
     "capability": [
      58.747843004374815,
      64.1123155056803
     ],
     "composite": {
      "A": [
       21.576286920084637,
       38.2033180855225
      ],
      "B": [
       19.014124679975207,
       35.0261728885226
      ],
      "C": [
       14.983532583392108,
       26.530082003835073
      ]
     },
     "seed": 35
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.67673072100749,
     "capability": 63.41348930844194,
     "intelligence": 37.15024789587638
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6813333333333333,
      "calibration": 89.09028590163935,
      "cc": 58.83962571040395,
      "cc_by_type": {
       "choice": 58.83962571040395
      },
      "cc_tiered": 60.666548963451085,
      "cc_tiered_open": 65.08523959814883,
      "cc_tiered_sealed": 56.24785832875334,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.83962571040395
     },
     "noul": {
      "accuracy": 0.5493333333333333,
      "calibration": 94.36563753351206,
      "cc": 9.866666666666667,
      "cc_by_type": {
       "noul": 9.866666666666667
      },
      "cc_tiered": 11.324016563147001,
      "cc_tiered_open": 22.648033126294003,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 9.866666666666667
     },
     "score": {
      "accuracy": 0.544,
      "calibration": 85.57426872787106,
      "cc": 36.35312382671484,
      "cc_by_type": {
       "score": 36.35312382671484
      },
      "cc_tiered": 39.46017816103106,
      "cc_tiered_open": 47.35108843691149,
      "cc_tiered_sealed": 31.56926788515063,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.35312382671484
     }
    },
    "raw_on_A": {
     "calibration": 83.97311382409126,
     "capability": 61.0776332772807,
     "intelligence": 38.182152730470136
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.086502951259888,
    "Emphasis on Accuracy 60:20:20": 23.485824898310423,
    "Emphasis on Cost 20:20:60": 26.536140479085216,
    "Emphasis on Speed 20:60:20": 32.79447793389916,
    "Intelligence only": 19.581330456126626
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "TypeSafe AI",
   "axes": {
    "calibration": 90.08195706553009,
    "cost": 54.815314343766296,
    "intelligence": 59.942981848430136,
    "speed": 90.98627348971067
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.040476900149031346,
      "ece_score": 91.90461997019374,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.90461997019374,
      "score_before_time_limit": 91.90461997019374,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.24128686327077753,
      "brier": 0.08160294906166222,
      "ece": 0.06619302949061674,
      "n": 373,
      "score": 86.76139410187666,
      "score_before_time_limit": 86.76139410187666,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.2782258064516129,
      "n": 372,
      "nrps": 0.06044260987193544,
      "score": 91.57985712451988,
      "score_before_time_limit": 91.57985712451988,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.053980123818833534
     }
    },
    "score": 90.08195706553009
   },
   "capability": 75.01246945698011,
   "composite_ci95": {
    "A": [
     67.73670723361865,
     71.21043212518939
    ],
    "B": [
     64.61448469474541,
     69.61304774005897
    ],
    "C": [
     55.864974598132086,
     71.21043212518939
    ]
   },
   "cost": {
    "basis": "TypeSafe AI public list price, USD 0.042/M input, output free: the \"Current models\" table of TypeSafe's models page (docs.typesafe.ai/models, fetched 16 Oct 2026 22:37:44Z, saved copy sha256 936da179...) lists jev-1.14.0 at \"$42 / $0.042\" per Btok / per Mtok, the same price as jev-1.13.0, and says \"You pay per input token. Output tokens are free.\"; Jev 1.14 is \"available from October 14, 2026\". Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4, Nimble, kettei and Celeris-1 Decision), through a scoped re-freeze of the frozen pricing module that adds this one key and nothing else, with a zero-difference proof for every other registry entry; jev-1.13.0 is priced through the same launch-list set at the same tariff. On the same page the alias jev-latest (\"the most recent stable, official release\") still points to jev-1.13.0, and jev-preview (\"the most recent release, official or not\") points to jev-1.14.0; the versioned id is listed with its price, accepted by the API and was the served model on every row. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (1,134,053 input tokens over the 1,485 answered items; the 71,566 reported output tokens are free under the tariff), divided by the 1,485 items of the common cost basis the scorer uses; the 15 requests refused with HTTP 400 report no usage and count as zero tokens and as wrong answers (TypeSafe's docs do not say whether refused requests are billed). USD 0.032074 per 1,000 decisions on the common basis (USD 0.031753 over all 1,500 items), 0.497x the official cost cap. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.032074 per 1,000 decisions (0.497x the official cost cap USD 0.064595), against USD 0.031753 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.031753484
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.032074226262626265
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jev 1.14.0 (TypeSafe AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted TypeSafe AI API (api.typesafe.ai), model `jev-1.14.0` (served model id `jev-1.14.0` on all 1,500 responses); latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of `v1.6-regular-20261012` through the unchanged `run_api_full_v161.py` and the unchanged `jevbench.adapters.typesafe`; the full-set exposure gate logged all 1,200 sealed items to provider `typesafe` before the first request (custody #20474). Native probabilities on all three question types. 15 of 1,500 requests returned HTTP 400 max_tokens_exceeded (the 15 long items of this draw; per TypeSafe's API reference the request is longer than the model's context length, 64k per request by default); the runner never re-sends a failed item, so they are scored as wrong in the full 1,500 denominator. TypeSafe documents an opt-in long-context mode (request header TypeSafe-Danger-Zone-Long-Context: true, up to 262k tokens at the same per-token price); this run used the default 64k context, as every hosted row of this draw used its default, so the 15 longest items were refused. 12 answered rows return probabilities that sum to 0.99 (the API rounds to two decimals); the unchanged scorer renormalises distributions within its 2e-2 tolerance, as it does for the published jev-1.13.0 raw, which has the same two-decimal property. Hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.09089256935101,
    "I_sealed": 53.79507112750927,
    "base": 59.942981848430136,
    "excess": 2.4562946714860345,
    "gap": 12.295821441841746,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.38533059643669,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 73.57659641396664,
       "judge": 84.17930125247199,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 52.73567977915805,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 66.1516673324583,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.0,
       "hard": 71.16105499438834,
       "judge": 74.77229926382469,
       "standard": 41.77777777777777
      }
     },
     "sealed|choice": {
      "cc": 70.9152814841789,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 65.13241586677461,
       "judge": 64.74045053868758,
       "standard": 81.96582999367222
      }
     },
     "sealed|noul": {
      "cc": 37.13412680988595,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 29.0909090909091,
       "judge": 32.30769230769231,
       "standard": 46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 53.33580508846296,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.47368421052632,
       "hard": 50.35293782899417,
       "judge": 53.192708333333314,
       "standard": 46.44724517906332
      }
     }
    }
   },
   "jevbench_score": 70.1541854578216,
   "key": "jev-1.14.0",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.27",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.typesafe.ai/v1/systemone#jev-1.14.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9540636042402827,
    "decisive": 283,
    "decisive_rate": 0.7587131367292225,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 16,
    "axes": {
     "calibration": 90.08195706553009,
     "cost": 54.815314343766296,
     "intelligence": 59.942981848430136,
     "speed": 90.96457246082724
    },
    "board_release": "v1.6.43",
    "capability": 75.01246945698011,
    "capability_api_board_rank": 10,
    "capability_open_board_rank": null,
    "capability_rank": 23,
    "composite_ci95": {
     "A": [
      67.7336997653583,
      71.20710829258444
     ],
     "B": [
      64.61229538497312,
      69.61050660584631
     ],
     "C": [
      55.86249334963948,
      71.20710829258444
     ]
    },
    "jevbench_score": 70.1509594948736,
    "measured_in": "v1.6.27",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 26,
    "ranks": {
     "A": 26,
     "B": 31,
     "C": 24,
     "capability": 43
    },
    "scores": {
     "A": 70.1509594948736,
     "B": 67.84038658562716,
     "C": 70.01769357840206
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.2238923824916128,
     "p50_s_raw": 0.2238923824916128,
     "p95_s_adjusted": 0.3576872305740834,
     "p95_s_raw": 0.3576872305740834
    },
    "views": {
     "Balanced 33:33:33": 65.33259265119166,
     "Emphasis on Accuracy 60:20:20": 63.06448377477604,
     "Emphasis on Cost 20:20:60": 60.67589899590322,
     "Emphasis on Speed 20:60:20": 73.63177120780584,
     "Intelligence only": 59.94298184843013
    }
   },
   "rank": 28,
   "ranked": true,
   "ranks": {
    "A": 28,
    "B": 31,
    "C": 25,
    "capability": 55
   },
   "repo": "https://docs.typesafe.ai/models",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.1541854578216,
    "B": 67.8428001241696,
    "C": 70.02091341298186
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22666933800792322,
    "p50_s_raw": 0.22666933800792322,
    "p95_s_adjusted": 0.35154414957505653,
    "p95_s_raw": 0.35154414957505653
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 48.444444444444436,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": -37.999999999999986
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 48.444444444444436
      },
      "calibration_gen": {
       "accuracy": 0.8415841584158416,
       "cc": 75.80772107786954,
       "cc_by_type": {
        "choice": 90.45599151643691,
        "noul": 76.0,
        "score": 60.96717171717172
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 75.80772107786954
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 83.08601440715262,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 85.18518518518519,
        "score": 70.57692307692307
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 83.08601440715262
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.62962962962963,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 90.88888888888889
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.62962962962963
      },
      "jaggedness": {
       "accuracy": 0.6388888888888888,
       "cc": 41.63374516451571,
       "cc_by_type": {
        "choice": 58.132820019249294,
        "noul": 13.513513513513509,
        "score": 53.254901960784316
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 41.63374516451571
      },
      "llm_router": {
       "accuracy": 0.805111821086262,
       "cc": 69.88977768858938,
       "cc_by_type": {
        "choice": 69.8864136655807,
        "noul": 85.56701030927836,
        "score": 54.2159090909091
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.88977768858938
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 4.824284408476845,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -50.0,
        "score": 26.328523328523325
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 4.824284408476845
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 40.78295167895502,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": -5.555555555555558,
        "score": 58.22720159339878
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.78295167895502
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 71.22659932659933,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 53.679797979797975
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 71.22659932659933
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 81.58273658273659,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100.0,
        "score": 65.76923076923077
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.58273658273659
      },
      "usecase": {
       "accuracy": 0.6772727272727272,
       "cc": 45.09244048326402,
       "cc_by_type": {
        "choice": 71.81372549019609,
        "noul": 8.333333333333325,
        "score": 55.130262626262635
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 45.09244048326402
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -3.8035668035668095,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": -50.0,
        "score": -23.315462315462334
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": 14.130670762928817,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 17.19999999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 14.130670762928817
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 75.16583301269719,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 74.3913043478261
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 75.16583301269719
      },
      "el": {
       "accuracy": 0.6,
       "cc": 44.21570453134698,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 99.49999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.21570453134698
      },
      "en": {
       "accuracy": 0.7767158992180713,
       "cc": 63.373471305868186,
       "cc_by_type": {
        "choice": 75.17726168828789,
        "noul": 56.11510791366907,
        "score": 58.82804431564761
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 63.373471305868186
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 72.85500431820361,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 28.35294117647056
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 72.85500431820361
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 31.91985428051002,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 94.66666666666667
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 31.91985428051002
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 74.20949432404541,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100.0,
        "score": 86.15789473684211
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 74.20949432404541
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 38.826815642458094,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 49.99999999999998
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 38.826815642458094
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 51.4564450877779,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -17.57142857142857
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 51.4564450877779
      },
      "ja": {
       "accuracy": 0.4,
       "cc": 0.3594740093548647,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": -21.733333333333338
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.3594740093548647
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 1.9232804232804217,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 50.21428571428571
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 1.9232804232804217
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.44444444444444,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.0
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.44444444444444
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -6.1826839826839874,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 33.399999999999984
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 42.02553867862059,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 61.31148696264975
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 42.02553867862059
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 73.3822938645681,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 51.96774193548389
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.3822938645681
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 22.968085106382983,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 65.5
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 22.968085106382983
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 27.670289855072465,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0.0,
        "score": 4.749999999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 27.670289855072465
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.66129032258064,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 97.49999999999999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.66129032258064
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 31.88888888888889,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 29.000000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.88888888888889
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7946666666666666,
       "cc": 73.47825254980003,
       "cc_by_type": {
        "choice": 73.47825254980003
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 73.47825254980003
      },
      "noul": {
       "accuracy": 0.72,
       "cc": 43.99999999999999,
       "cc_by_type": {
        "noul": 43.99999999999999
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 43.99999999999999
      },
      "score": {
       "accuracy": 0.6933333333333334,
       "cc": 56.740655654013075,
       "cc_by_type": {
        "score": 56.740655654013075
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 56.740655654013075
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": 6.09803921568628,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -75.0,
        "score": 25.294117647058833
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.09803921568628
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 31.665734426604,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 55.55555555555556,
        "score": -4.6577311577311376
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 31.665734426604
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 38.40650406504064,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 13.999999999999979
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 38.40650406504064
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -28.41544834307992,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -3.3749999999999947
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 30.187801932367147,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 67.37499999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 30.187801932367147
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": 13.061844863731627,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 0.0,
        "score": -10.500000000000087
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 13.061844863731627
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 49.6704014939309,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 54.33333333333332
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 49.6704014939309
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 29.958895116901484,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50.0,
        "score": 60.78260869565217
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 29.958895116901484
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 18.722222222222225,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 78.66666666666667
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.722222222222225
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 38.48014405294919,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 0.0,
        "score": 41.13636363636364
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 38.48014405294919
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 43.90517578889672,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 54.45454545454547
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.90517578889672
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -8.535842293906812,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": 4.499999999999993
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 71.50523756079312,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 54.51571268237934
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 71.50523756079312
      },
      "model_routing": {
       "accuracy": 0.8294930875576036,
       "cc": 70.61857413419914,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 90.90909090909092,
        "score": 40.81676136363635
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 70.61857413419914
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 8.93466561288623,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 61.3529411764706
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 8.93466561288623
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 31.887853248577482,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 62.423076923076934
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 31.887853248577482
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 9.766154737757171,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": 38.41176470588235
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 9.766154737757171
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 9.087439613526568,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 51.89999999999999
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 9.087439613526568
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 83.06140350877193,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 99.18421052631578
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 83.06140350877193
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.10088425105317,
      90.38371712636082
     ],
     "I": [
      54.572148574825896,
      63.98122322055633
     ],
     "capability": [
      70.67669877479854,
      76.35848785536061
     ],
     "composite": {
      "A": [
       67.73670723361865,
       71.21043212518939
      ],
      "B": [
       64.61448469474541,
       69.61304774005897
      ],
      "C": [
       55.864974598132086,
       71.21043212518939
      ]
     },
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.08195706553009,
     "capability": 75.01246945698011,
     "intelligence": 59.942981848430136
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7946666666666666,
      "calibration": 91.90461997019374,
      "cc": 73.47825254980003,
      "cc_by_type": {
       "choice": 73.47825254980003
      },
      "cc_tiered": 75.1503060403078,
      "cc_tiered_open": 79.38533059643669,
      "cc_tiered_sealed": 70.9152814841789,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.47825254980003
     },
     "noul": {
      "accuracy": 0.72,
      "calibration": 86.76139410187666,
      "cc": 43.99999999999999,
      "cc_by_type": {
       "noul": 43.99999999999999
      },
      "cc_tiered": 44.934903294522,
      "cc_tiered_open": 52.73567977915805,
      "cc_tiered_sealed": 37.13412680988595,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.99999999999999
     },
     "score": {
      "accuracy": 0.6933333333333334,
      "calibration": 91.57985712451988,
      "cc": 56.740655654013075,
      "cc_by_type": {
       "score": 56.740655654013075
      },
      "cc_tiered": 59.74373621046063,
      "cc_tiered_open": 66.1516673324583,
      "cc_tiered_sealed": 53.33580508846296,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.740655654013075
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.33632339558339,
    "Emphasis on Accuracy 60:20:20": 63.066569447746595,
    "Emphasis on Cost 20:20:60": 60.67782966754747,
    "Emphasis on Speed 20:60:20": 73.64030150613897,
    "Intelligence only": 59.94298184843013
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kadirbekovvv",
   "axes": {
    "calibration": 55.43211615661533,
    "cost": null,
    "intelligence": 18.416504152111088,
    "speed": 90.40620976128842
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.26331740063992715,
      "ece_score": 47.33651987201457,
      "mean_tvd": null,
      "n": 729,
      "n_ece_hard": 296,
      "n_tvd": 0,
      "score": 47.273404512185216,
      "score_before_time_limit": 47.33651987201457,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.12568306010928965,
      "brier": 0.2429692790949114,
      "ece": 0.21804147724652553,
      "n": 366,
      "score": 56.39170455069489,
      "score_before_time_limit": 56.39170455069489,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8375350140056023,
      "interval90_width": 0.6150793650793649,
      "n": 357,
      "nrps": 0.19607308788933592,
      "score": 62.631239406965896,
      "score_before_time_limit": 62.631239406965896,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.2756510619856731
     }
    },
    "score": 55.43211615661533
   },
   "capability": 36.92431015436321,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base (the module RAISES 'base model lacks an explicit market-listing decision'; the artifact's own metadata names this base); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's in-process adapter (input 681,813, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jevora-9b-v1 (kadirbekovvv / doxanocap, Qwen3.5-9B-Base derivative)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's in-process adapter and native decision head (no server).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 21.935398847280897,
    "I_sealed": 14.897609456941275,
    "base": 18.416504152111088,
    "excess": -2.801737380016089,
    "gap": 7.037789390339622,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 37.29135872265705,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 30.166719094054738,
       "judge": 48.58272907053395,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 28.514837819185644,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -4.347826086956519,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.56118625108944,
       "hard": 3.2474489744614754,
       "judge": -25.16569669784634,
       "standard": -22.34160066754731
      }
     },
     "sealed|choice": {
      "cc": 27.18704645330748,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 54.2566709021601,
       "hard": 21.938244477853576,
       "judge": 21.547502448579845,
       "standard": 32.609154186880424
      }
     },
     "sealed|noul": {
      "cc": 17.50578191751635,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 51.72413793103448,
       "hard": 5.454545454545445,
       "judge": 16.92307692307693,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 2.429989968802293,
       "hard": -40.50178363761847,
       "judge": -25.860796025760123,
       "standard": 10.253044034947001
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "jevora-9b-v1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "kadirbekovvv/jevora-9b-v1@c644b9c7a3b67b7acc3279523e3bee69a4be473d",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.728125,
    "decisive": 320,
    "decisive_rate": 0.8743169398907104,
    "n": 375,
    "supported": true,
    "valid": 366
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 55.45315460989178,
     "cost": null,
     "intelligence": 18.54232988347315,
     "speed": 90.58228710164383
    },
    "board_release": "v1.6.43",
    "capability": 36.99774224668246,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27591127399355175,
     "p50_s_raw": 0.06295563699677587,
     "p95_s_adjusted": 0.3169580915942788,
     "p95_s_raw": 0.0834790457971394
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/kadirbekovvv/jevora-9b-v1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27972743902355435,
    "p50_s_raw": 0.06486371951177716,
    "p95_s_adjusted": 0.3255696814507246,
    "p95_s_raw": 0.0877848407253623
   },
   "status": {
    "answered_ok": 1452,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5,
       "cc": -46.83461706671449,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": 50.0,
        "score": -223.83718453347683
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.33663366336633666,
       "cc": -3.5284110215130844,
       "cc_by_type": {
        "choice": 10.92258748674444,
        "noul": 43.99999999999999,
        "score": -65.50782055128369
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 3.1172240518339493,
       "cc_by_type": {
        "choice": 28.45528455284553,
        "noul": 33.33333333333333,
        "score": -52.43694573067701
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 3.1172240518339493
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 53.62620746968167,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": 19.999999999999996,
        "score": 69.45005098047356
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 53.62620746968167
      },
      "jaggedness": {
       "accuracy": 0.37777777777777777,
       "cc": -6.695722360583766,
       "cc_by_type": {
        "choice": 22.040423484119355,
        "noul": -13.513513513513509,
        "score": -28.614077052357146
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6741214057507987,
       "cc": 51.730031018571616,
       "cc_by_type": {
        "choice": 47.697455313903326,
        "noul": 73.1958762886598,
        "score": 34.2967614531517
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 51.730031018571616
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -38.364547830866904,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -50.0,
        "score": -66.12457132765226
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.4727272727272727,
       "cc": 5.811727778234681,
       "cc_by_type": {
        "choice": 35.441799804369104,
        "noul": 8.333333333333325,
        "score": -26.339949802998387
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 5.811727778234681
      },
      "negation_pair_gen": {
       "accuracy": 0.3488372093023256,
       "cc": -24.610571890241573,
       "cc_by_type": {
        "choice": 10.000000000000002,
        "noul": -13.33333333333333,
        "score": -70.49838233739139
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.2708333333333333,
       "cc": 6.790945406125167,
       "cc_by_type": {
        "choice": 6.790945406125167
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 6.790945406125167
      },
      "perturb_pair": {
       "accuracy": 0.6875,
       "cc": 59.76765762955464,
       "cc_by_type": {
        "choice": 48.94894894894895,
        "noul": 100.0,
        "score": 30.354023939714992
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 59.76765762955464
      },
      "usecase": {
       "accuracy": 0.38636363636363635,
       "cc": 1.345314494671392,
       "cc_by_type": {
        "choice": 25.245098039215712,
        "noul": -12.5,
        "score": -8.709154555201536
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 1.345314494671392
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -17.73475850312556,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0.0,
        "score": -77.01379931890048
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 38.634690940151195,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": -63.358599990606315
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 38.634690940151195
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 58.46529563147624,
       "cc_by_type": {
        "choice": 44.24778761061947,
        "noul": 100.0,
        "score": 31.14809928380925
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 58.46529563147624
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 16.196083473787407,
       "cc_by_type": {
        "choice": -0.5586592178770929,
        "noul": 0.0,
        "score": 49.14690963923931
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.196083473787407
      },
      "en": {
       "accuracy": 0.46481320590790615,
       "cc": 13.665211396900302,
       "cc_by_type": {
        "choice": 28.167828071417194,
        "noul": 25.899280575539564,
        "score": -13.07147445625585
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 13.665211396900302
      },
      "es": {
       "accuracy": 0.6451612903225806,
       "cc": 38.89156095653784,
       "cc_by_type": {
        "choice": 41.27243066884176,
        "noul": 100.0,
        "score": -24.59774779922823
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 38.89156095653784
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -43.89799635701275,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": -100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": 20.45477668736496,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 3.717271238565456
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 20.45477668736496
      },
      "hi": {
       "accuracy": 0.09523809523809523,
       "cc": -121.9593465935224,
       "cc_by_type": {
        "choice": -17.31843575418994,
        "noul": -75.0,
        "score": -273.5596040263772
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 65.42216278743965,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 10.296106444937813
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 65.42216278743965
      },
      "ja": {
       "accuracy": 0.48,
       "cc": -0.003854126466692994,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -19.999999999999996,
        "score": -8.526256581147496
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 19.542302638686518,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 3.0713523605039983
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 19.542302638686518
      },
      "mixed": {
       "accuracy": 0.6428571428571429,
       "cc": 49.22501272004511,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 33.33333333333333,
        "score": 64.34170482680202
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 49.22501272004511
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 33.90898539003356,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 100.0,
        "score": -46.324991881847396
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.90898539003356
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 22.898515881046155,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 19.999999999999996,
        "score": -28.452145966679552
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 22.898515881046155
      },
      "pt": {
       "accuracy": 0.5357142857142857,
       "cc": 38.58246838017991,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 100.0,
        "score": -10.00392072864118
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 38.58246838017991
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 6.878374932822606,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": 19.999999999999996,
        "score": 28.294699266552925
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 6.878374932822606
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -33.32266237154819,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0.0,
        "score": -91.27233494073153
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.2857142857142857,
       "cc": -44.56124096628158,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -69.16759386658669
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -59.687737369534545,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": -169.45869233459234
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.46,
       "cc": 30.251248588759843,
       "cc_by_type": {
        "choice": 30.251248588759843
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 30.251248588759843
      },
      "noul": {
       "accuracy": 0.6213333333333333,
       "cc": 24.26666666666666,
       "cc_by_type": {
        "noul": 24.26666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 24.26666666666666
      },
      "score": {
       "accuracy": 0.352,
       "cc": -15.561798891550138,
       "cc_by_type": {
        "score": -15.561798891550138
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.18181818181818182,
       "cc": -53.49933930121218,
       "cc_by_type": {
        "choice": 4.0,
        "noul": -50.0,
        "score": -114.49801790363652
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -27.327813582184664,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": -33.333333333333336,
        "score": -36.84886517719581
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 67.16319530556196,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 84.0099111199379
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 67.16319530556196
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 28.345389256969124,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": 33.5741794668138
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 28.345389256969124
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -3.2842516228736685,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -33.333333333333336,
        "score": -4.0556534193456395
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.21052631578947367,
       "cc": -61.25795271724025,
       "cc_by_type": {
        "choice": -13.207547169811319,
        "noul": -66.66666666666667,
        "score": -103.89964431524277
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.43333333333333335,
       "cc": -1.4206709993739324,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -14.28571428571429,
        "score": -40.956690869270254
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -8.466083304599605,
       "cc_by_type": {
        "choice": -25.435540069686407,
        "noul": 0.0,
        "score": 0.03729015588759266
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 7.990436824383561,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -19.999999999999996,
        "score": 48.137977139817345
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 7.990436824383561
      },
      "knowledge_graphs": {
       "accuracy": 0.18181818181818182,
       "cc": -21.352245247704335,
       "cc_by_type": {
        "choice": -28.479657387580303,
        "noul": -33.333333333333336,
        "score": -2.2437450221993727
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 3.569853000991013,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 11.111111111111116,
        "score": -77.1457381546497
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 3.569853000991013
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 16.028989510323616,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 33.33333333333333,
        "score": -52.98830028623345
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 16.028989510323616
      },
      "llm_guardrails": {
       "accuracy": 0.3333333333333333,
       "cc": -12.070180855485154,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -19.999999999999996,
        "score": -47.11265389083166
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.7050691244239631,
       "cc": 53.53856111416466,
       "cc_by_type": {
        "choice": 57.92207792207793,
        "noul": 87.87878787878789,
        "score": 14.814817541628155
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 53.53856111416466
      },
      "moderation_trust_safety": {
       "accuracy": 0.3333333333333333,
       "cc": 39.592028247842826,
       "cc_by_type": {
        "choice": -3.646833013435703,
        "noul": 100.0,
        "score": 22.42291775696419
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 39.592028247842826
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 17.99421275407803,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 20.742155439578582
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 17.99421275407803
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -38.314615283872065,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -50.0,
        "score": -91.05222023585264
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": 6.055643005647522,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 0.0,
        "score": 5.123450756073
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 6.055643005647522
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 67.96574075461801,
       "cc_by_type": {
        "choice": 57.295373665480426,
        "noul": 100.0,
        "score": 46.601848598373564
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 67.96574075461801
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      49.71252057656553,
      59.65793702420633
     ],
     "I": [
      13.85411913413391,
      23.55473519133409
     ],
     "capability": [
      32.36265845391065,
      41.127021044667465
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 37
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 55.43211615661533,
     "capability": 36.92431015436321,
     "intelligence": 18.416504152111088
    },
    "per_type": {
     "choice": {
      "accuracy": 0.46,
      "calibration": 47.273404512185216,
      "cc": 30.251248588759843,
      "cc_by_type": {
       "choice": 30.251248588759843
      },
      "cc_tiered": 32.23920258798226,
      "cc_tiered_open": 37.29135872265705,
      "cc_tiered_sealed": 27.18704645330748,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 30.251248588759843
     },
     "noul": {
      "accuracy": 0.6213333333333333,
      "calibration": 56.39170455069489,
      "cc": 24.26666666666666,
      "cc_by_type": {
       "noul": 24.26666666666666
      },
      "cc_tiered": 23.010309868350998,
      "cc_tiered_open": 28.514837819185644,
      "cc_tiered_sealed": 17.50578191751635,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.26666666666666
     },
     "score": {
      "accuracy": 0.352,
      "calibration": 62.631239406965896,
      "cc": -15.561798891550138,
      "cc_by_type": {
       "score": -15.561798891550138
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 58.65667414349311,
     "capability": 37.50098153070991,
     "intelligence": 16.345288917926695
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.029333333333333322,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.02400000000000002,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.04800000000000004,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kirp",
   "axes": {
    "calibration": 86.88996927941005,
    "cost": 64.21228687378368,
    "intelligence": 40.7504160092812,
    "speed": 87.81344131912249
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06574074427929287,
      "ece_score": 86.85185114414142,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 86.85185114414142,
      "score_before_time_limit": 86.85185114414142,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3351206434316354,
      "brier": 0.14817864619281185,
      "ece": 0.08688407846981186,
      "n": 373,
      "score": 82.62318430603763,
      "score_before_time_limit": 82.62318430603763,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.956989247311828,
      "interval90_width": 0.5188172043010754,
      "n": 372,
      "nrps": 0.10223206878863621,
      "score": 91.1948723880511,
      "score_before_time_limit": 91.1948723880511,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03693524172517095
     }
    },
    "score": 86.88996927941005
   },
   "capability": 63.820192644345624,
   "composite_ci95": {
    "A": [
     30.8407381419149,
     51.026924198007464
    ],
    "B": [
     27.244880123361554,
     46.54197118164839
    ],
    "C": [
     21.417179265218678,
     35.43536402639406
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the author's server, KEPT AS SENT: output_tokens is 1 per answered question (the label position the engine counts, not generated text); this adds about USD 0.0002 per 1,000 decisions at the reference output rate. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.015593 per 1,000 decisions (0.241x the official cost cap USD 0.064595), against USD 0.015437 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01543686
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01559278787878788
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JPT-4B (kirp, Qwen3.5-4B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's llm2jev engine on SGLang: one prefill, next-token distribution over the option labels, no generation.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 50.49454797607648,
    "I_sealed": 31.75033894489564,
    "base": 41.12244346048606,
    "excess": 8.904682260825133,
    "gap": 18.744209031180844,
    "penalty": 0.9909531773917487,
    "per_type_split": {
     "open|choice": {
      "cc": 63.186141632614884,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 40.069013112491376,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 48.2284891831232,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.18914834159548,
       "hard": 39.875244434411385,
       "judge": 44.47460178000418,
       "standard": 62.08548020598924
      }
     },
     "sealed|choice": {
      "cc": 57.083332727349514,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 49.52006476234531,
       "judge": 55.92556317335946,
       "standard": 65.82999367222105
      }
     },
     "sealed|noul": {
      "cc": 8.07096660416269,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 5.454545454545445,
       "judge": -10.769230769230765,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 30.096717503174723,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 44.10049974856993,
       "hard": 22.500740414784172,
       "judge": 31.671419285857695,
       "standard": 35.92472788323378
      }
     }
    }
   },
   "jevbench_score": 42.16748030386713,
   "key": "jpt-4b",
   "last_measured_on": "2026-10-16",
   "licence": "CC-BY-NC-4.0 (weights, non-commercial)",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "kirp/jpt-4b@05285d2d4f0cd954666ed88095f511c80665bc4c",
   "noul_decisive": {
    "acc_among_decisive": 0.8991935483870968,
    "decisive": 248,
    "decisive_rate": 0.6648793565683646,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.88996927941005,
     "cost": 64.21228687378368,
     "intelligence": 40.7504160092812,
     "speed": 87.95549105905897
    },
    "board_release": "v1.6.43",
    "capability": 63.820192644345624,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 42,
    "capability_rank": 65,
    "composite_ci95": {
     "A": [
      30.849201560578464,
      51.04227850724084
     ],
     "B": [
      27.250163615926105,
      46.552190181580386
     ],
     "C": [
      21.4230566392906,
      35.44602674113947
     ]
    },
    "jevbench_score": 42.17979190383397,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 49,
    "rank": 76,
    "ranks": {
     "A": 76,
     "B": 82,
     "C": 80,
     "capability": 107
    },
    "scores": {
     "A": 42.17979190383397,
     "B": 37.943136667208464,
     "C": 29.29152215544026
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27590461224317553,
     "p50_s_raw": 0.06295230612158775,
     "p95_s_adjusted": 0.5803525988012552,
     "p95_s_raw": 0.2151762994006276
    },
    "views": {
     "Balanced 33:33:33": 38.706768663218696,
     "Emphasis on Accuracy 60:20:20": 33.02644805072411,
     "Emphasis on Cost 20:20:60": 40.19401225266592,
     "Emphasis on Speed 20:60:20": 44.74726776912738,
     "Intelligence only": 27.06799772975727
    }
   },
   "rank": 86,
   "ranked": true,
   "ranks": {
    "A": 86,
    "B": 93,
    "C": 92,
    "capability": 125
   },
   "repo": "https://huggingface.co/kirp/jpt-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 42.16748030386713,
    "B": 37.935165946028185,
    "C": 29.282972433241063
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2816717634908855,
    "p50_s_raw": 0.06583588174544275,
    "p95_s_adjusted": 0.5873710740823298,
    "p95_s_raw": 0.21868553704116492
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 45.51030495110692,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100.0,
        "score": -21.802418480012587
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 45.51030495110692
      },
      "calibration_gen": {
       "accuracy": 0.40594059405940597,
       "cc": 6.942600718140113,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -12.0,
        "score": 12.361206184112806
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 6.942600718140113
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 30.524274653794333,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -11.111111111111116,
        "score": 35.204260275746144
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 30.524274653794333
      },
      "fairness": {
       "accuracy": 0.8541666666666666,
       "cc": 80.0574932650096,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 80.17247979502879
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 80.0574932650096
      },
      "jaggedness": {
       "accuracy": 0.4722222222222222,
       "cc": 8.716662462559588,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -18.918918918918916,
        "score": 2.8167407628055585
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 8.716662462559588
      },
      "llm_router": {
       "accuracy": 0.7699680511182109,
       "cc": 64.43530352033419,
       "cc_by_type": {
        "choice": 63.546711279387154,
        "noul": 71.13402061855669,
        "score": 58.62517866305872
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 64.43530352033419
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -19.145184391822436,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75.0,
        "score": 16.533518989481156
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6690909090909091,
       "cc": 40.9417458426497,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 11.111111111111116,
        "score": 45.949535611490745
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.9417458426497
      },
      "negation_pair_gen": {
       "accuracy": 0.5813953488372093,
       "cc": 17.585802811544415,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -26.66666666666667,
        "score": 6.090741767966568
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 17.585802811544415
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 75.72834158389229,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100.0,
        "score": 57.2150547817069
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 75.72834158389229
      },
      "usecase": {
       "accuracy": 0.5545454545454546,
       "cc": 30.51414077961618,
       "cc_by_type": {
        "choice": 58.33333333333336,
        "noul": 0.0,
        "score": 33.20908900551518
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 30.51414077961618
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 13.779892768838891,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50.0,
        "score": -13.422226455388087
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 13.779892768838891
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 57.81691011189755,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 40.117397002359326
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 57.81691011189755
      },
      "de": {
       "accuracy": 0.875,
       "cc": 74.83045721744055,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75.0,
        "score": 65.4205751921447
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 74.83045721744055
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 32.71944928714865,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -33.333333333333336,
        "score": 98.53078734003067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 32.71944928714865
      },
      "en": {
       "accuracy": 0.6159860990443093,
       "cc": 36.19377416348588,
       "cc_by_type": {
        "choice": 57.823311895144045,
        "noul": 19.424460431654666,
        "score": 31.33355016365892
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 36.19377416348588
      },
      "es": {
       "accuracy": 0.9354838709677419,
       "cc": 75.00594678954843,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 34.805768590505025
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 75.00594678954843
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 59.045268832756356,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 76.58935841083739
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 59.045268832756356
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 48.18668967185474,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 65.7365396037995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 48.18668967185474
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 11.105436929386585,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50.0,
        "score": 33.5956403970983
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 11.105436929386585
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 69.7459706377814,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 9.237911913344199
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 69.7459706377814
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 29.893084358860033,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 19.999999999999996,
        "score": 41.16455887483268
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.893084358860033
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -2.930607641753516,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -8.79182292526055
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 93.59160328181015,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 97.44147651209715
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 93.59160328181015
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 19.559769597334363,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 43.960694073388375
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.559769597334363
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 27.11064787261347,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -19.999999999999996,
        "score": 24.18425000802239
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 27.11064787261347
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 62.70420351353226,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 59.93347088237638
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.70420351353226
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 21.200430571063105,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 60.19703639404037
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 21.200430571063105
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 35.867549234439366,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50.0,
        "score": 1.08090857288331
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 35.867549234439366
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 70.75565094393932,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 76.78308186407601
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 70.75565094393932
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -2.0732738607933356,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -22.039030621928024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6826666666666666,
       "cc": 59.011844849690966,
       "cc_by_type": {
        "choice": 59.011844849690966
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.011844849690966
      },
      "noul": {
       "accuracy": 0.5946666666666667,
       "cc": 18.933333333333337,
       "cc_by_type": {
        "noul": 18.933333333333337
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.933333333333337
      },
      "score": {
       "accuracy": 0.552,
       "cc": 34.07730164309798,
       "cc_by_type": {
        "score": 34.07730164309798
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 34.07730164309798
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -8.588501027763224,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -75.0,
        "score": 13.234496916710325
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 25.604935975995005,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 33.33333333333333,
        "score": -0.6179042873358798
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 25.604935975995005
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 62.01973277404002,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 18.5795235253721
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.01973277404002
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -21.240602652279154,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": 6.453630639653774
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 20.612938409878996,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 38.65040943253555
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 20.612938409878996
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -43.63239240289681,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -66.66666666666667,
        "score": -113.91604513321872
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 51.14704373786173,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 39.99575306232466
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.14704373786173
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -8.074338669114693,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100.0,
        "score": 38.49475402749912
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 11.222550992581288,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 56.16765297774386
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.222550992581288
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 10.878008756922313,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0.0,
        "score": 9.72182070331511
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 10.878008756922313
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 39.76323274592266,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 19.80649410340106
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 39.76323274592266
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -37.43831056220761,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": 16.717326377893283
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 24.3873817264206,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 27.711089517073706
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 24.3873817264206
      },
      "model_routing": {
       "accuracy": 0.815668202764977,
       "cc": 67.39221756780174,
       "cc_by_type": {
        "choice": 77.7922077922078,
        "noul": 78.78787878787878,
        "score": 45.59656612331862
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.39221756780174
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 0.5110011263103331,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 47.59826249601353
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.5110011263103331
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 49.16687512897187,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 47.59347589759341
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 49.16687512897187
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 22.506551648724535,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 47.076305192479005
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.506551648724535
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 20.443709196711993,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0.0,
        "score": 19.302142082889596
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 20.443709196711993
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 92.90014978449283,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 89.37660593710837
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.90014978449283
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.5481127516258,
      87.7665054437864
     ],
     "I": [
      35.92555449500924,
      44.16081939325531
     ],
     "capability": [
      58.87655522688167,
      65.22805681139722
     ],
     "composite": {
      "A": [
       30.8407381419149,
       51.026924198007464
      ],
      "B": [
       27.244880123361554,
       46.54197118164839
      ],
      "C": [
       21.417179265218678,
       35.43536402639406
      ]
     },
     "seed": 38
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.88996927941005,
     "capability": 63.820192644345624,
     "intelligence": 40.7504160092812
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6826666666666666,
      "calibration": 86.85185114414142,
      "cc": 59.011844849690966,
      "cc_by_type": {
       "choice": 59.011844849690966
      },
      "cc_tiered": 60.134737179982196,
      "cc_tiered_open": 63.186141632614884,
      "cc_tiered_sealed": 57.083332727349514,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.011844849690966
     },
     "noul": {
      "accuracy": 0.5946666666666667,
      "calibration": 82.62318430603763,
      "cc": 18.933333333333337,
      "cc_by_type": {
       "noul": 18.933333333333337
      },
      "cc_tiered": 24.069989858327034,
      "cc_tiered_open": 40.069013112491376,
      "cc_tiered_sealed": 8.07096660416269,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.933333333333337
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 91.1948723880511,
      "cc": 34.07730164309798,
      "cc_by_type": {
       "score": 34.07730164309798
      },
      "cc_tiered": 39.16260334314896,
      "cc_tiered_open": 48.2284891831232,
      "cc_tiered_sealed": 30.096717503174723,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 34.07730164309798
     }
    },
    "raw_on_A": {
     "calibration": 86.19822310825856,
     "capability": 63.499626434412406,
     "intelligence": 40.80102976056625
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 38.69294605930085,
    "Emphasis on Accuracy 60:20:20": 33.020409027270375,
    "Emphasis on Cost 20:20:60": 40.185067909231535,
    "Emphasis on Speed 20:60:20": 44.71402830802048,
    "Intelligence only": 27.06799772975727
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Supersonic Labs / @supersonicai (Hugging Face SupersonicLabs)",
   "axes": {
    "calibration": 17.26803648238681,
    "cost": 89.74859303626371,
    "intelligence": 2.4412732607743255,
    "speed": 77.74663940674506
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.49162634593303195,
      "ece_score": 1.67473081339361,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 304,
      "n_tvd": 0,
      "score": 1.67473081339361,
      "score_before_time_limit": 1.67473081339361,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.27419354838709675,
      "brier": 0.45131804975332956,
      "ece": 0.4322726807691505,
      "n": 372,
      "score": 13.545463846169902,
      "score_before_time_limit": 13.545463846169902,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.5887096774193549,
      "interval90_width": 0.3958333333333335,
      "n": 372,
      "nrps": 0.26832170424806157,
      "score": 36.583914787596925,
      "score_before_time_limit": 36.583914787596925,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.5187092957892186
     }
    },
    "score": 17.26803648238681
   },
   "capability": 9.854654871580568,
   "composite_ci95": {
    "A": [
     0.0007030229261073844,
     0.1394909016822168
    ],
    "B": [
     0.0004538884390108904,
     0.10093093032471417
    ],
    "C": [
     0.0004882103653523503,
     0.09686868172376169
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public bookable tariff for these weights. The frozen pricing_v15 path is asserted with ONE authorised, scoped correction: CoS deputy decision #18761 added the declared base jhu-clsp/mmBERT-small to pricing_v15.UNLISTED_BASE_MODELS, exactly as openbmb/MiniCPM5-2B and answerdotai/ModernBERT-base already are, so reference() returns no market reference for it rather than raising and the ancestry can be carried in base_model instead of being dropped to null. The addition changes no other row: it was proved behaviour-identical over every registry entry on the box before it was installed (r108/receipts/PRICING-EQUIV-R108.json) and independently reviewed (r108/receipts/REVIEW-PRICING-R108.json). Florian's standing 24 Sep rule then gives a clearly labelled size-class estimate: the base-size-encoder line the PUBLISHED v1.6 field already carries - USD 0.005 per 1M input and 0.0 per 1M output, the line modernbert-jev-v2, tacet-sonata and watt-flash-0.1 carry there (no row of THIS draw carries it, and lev-350m carries 0.01/0.0, not this line) (four base-size encoders in the dated 25 Sep 2026 DeepInfra snapshot sha256 84eb7926... all price at exactly that) - so no new release-lane pricing line is created. x this row's own measured tokens: usage.input_tokens comes from the author's OWN audit function engine.encoding_info() and usage.output_tokens is 0 by construction (one encoder forward pass read at marker positions, nothing generated). Estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.002196 per 1,000 decisions (0.034x the official cost cap USD 0.064595), against USD 0.002174 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0021744466666666668
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0021964107744107744
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Julia-1 (Supersonic Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s into logs/CONDITIONS-julia-1.jsonl (11 samples, 10.87-13.78, mean 12.5). The author ships NO server, so the HTTP surface is OURS: r107/src/julia_shim_r107.py, a pure transport that makes no mapping decision, because the author's own julia.typed.predict_typed already speaks JevBench's typed-question shape exactly (choice keyed by the item's own criteria keys, score keyed '0'..'n-1' with score = sum(i*p_i), noul = p(true)). Engine built with the author's own inference-policy.json (max_length 8192, head_length 512, strict_encoding true), torch backend, device cpu, float32 (their julia_config weight_dtype), thread count left at their own default of 4 - all four read back from the run's own /health rather than asserted. Run inside a loopback-only network namespace whose outbound probe failed as required, offline (HF_HUB_OFFLINE=1), credential-free, nice 5, in a systemd scope with MemoryMax=8G and no swap. Speed is therefore a loaded-shared-CPU figure and a lower bound for this model on a dedicated core or a GPU; the standard self-hosted x2 + 0.15 s adjustment is applied on top and the raw p50/p95 are kept. Measured for this release on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...) under CoS deputy decision #18209, which admitted this row only after an independent source review and a synthetic-only smoke, both of which were met in that order before the first sealed item. DISCLOSED: the author's own strict_encoding policy REFUSES rather than truncates, so 17 of the 1,500 items (15 over their lossless context budget, 2 with an option above their 48-token option contract) are honest item-level misses recorded as HTTP 422 - predicted exactly, class split included, before the run.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 2.994076934245856,
    "I_sealed": 1.888469587302795,
    "base": 2.4412732607743255,
    "excess": -8.733919423412651,
    "gap": 1.105607346943061,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 8.982230802737568,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.7027027027027115,
       "hard": 13.180245360176164,
       "judge": 5.075807514831907,
       "standard": 9.585600669736293
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -4.347826086956519,
       "judge": 0.0,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -4.595553956916043,
       "hard": -21.831788639496995,
       "judge": -76.77226748673378,
       "standard": -19.64084658486791
      }
     },
     "sealed|choice": {
      "cc": 5.6654087619083855,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 12.325285895806868,
       "hard": 0.6013646351335588,
       "judge": 11.85112634671892,
       "standard": 3.1849820712929935
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -41.379310344827594,
       "hard": -41.81818181818182,
       "judge": -50.76923076923077,
       "standard": -19.402985074626866
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -36.27109595021023,
       "hard": -40.51450244498809,
       "judge": -18.32435058181421,
       "standard": -13.222659125532354
      }
     }
    }
   },
   "jevbench_score": 0.01939975687732831,
   "key": "julia-1",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 (Hugging Face cardData.license and README; the pinned tree contains no LICENSE file)",
   "listing": "ranked",
   "measured_in": "v1.6.15",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "SupersonicLabs/Julia-1@a85b127321d580d65176c89ced8273f305745d85",
   "noul_decisive": {
    "acc_among_decisive": 0.45555555555555555,
    "decisive": 270,
    "decisive_rate": 0.7258064516129032,
    "n": 375,
    "supported": true,
    "valid": 372
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 17.26803648238681,
     "cost": 89.74859303626371,
     "intelligence": 2.4412732607743255,
     "speed": 79.42119248319958
    },
    "board_release": "v1.6.43",
    "capability": 9.854654871580568,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 138,
    "capability_rank": 168,
    "composite_ci95": {
     "A": [
      0.0007031616572555455,
      0.13962672310195573
     ],
     "B": [
      0.000453934696458772,
      0.10098779426488583
     ],
     "C": [
      0.0004883067064274621,
      0.09696300215413593
     ]
    },
    "jevbench_score": 0.019410466159695165,
    "measured_in": "v1.6.15",
    "method": "v1.6",
    "open_board_rank": 186,
    "rank": 230,
    "ranks": {
     "A": 230,
     "B": 230,
     "C": 230,
     "capability": 240
    },
    "scores": {
     "A": 0.019410466159695165,
     "B": 0.013230972842696788,
     "C": 0.0134794903886772
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7337018699967303,
     "p50_s_raw": 0.2918509349983651,
     "p95_s_adjusted": 1.5572599903855007,
     "p95_s_raw": 0.7036299951927504
    },
    "views": {
     "Balanced 33:33:33": 0.016503253650422215,
     "Emphasis on Accuracy 60:20:20": 0.009515909044619377,
     "Emphasis on Cost 20:20:60": 0.026160186027009726,
     "Emphasis on Speed 20:60:20": 0.02599486708790751,
     "Intelligence only": 0.005819814930094012
    }
   },
   "rank": 252,
   "ranked": true,
   "ranks": {
    "A": 252,
    "B": 252,
    "C": 252,
    "capability": 262
   },
   "repo": "https://huggingface.co/SupersonicLabs/Julia-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.01939975687732831,
    "B": 0.013226991120613374,
    "C": 0.01347205338703355
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8151886820036452,
    "p50_s_raw": 0.3325943410018226,
    "p95_s_adjusted": 2.0609996498023975,
    "p95_s_raw": 0.9554998249011987
   },
   "status": {
    "answered_ok": 1483,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.2727272727272727,
       "cc": -27.879433144127862,
       "cc_by_type": {
        "choice": 0.0,
        "noul": 0.0,
        "score": -83.63829943238359
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.3564356435643564,
       "cc": -3.761010359494321,
       "cc_by_type": {
        "choice": 4.559915164369044,
        "noul": 4.0000000000000036,
        "score": -19.842946242852012
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.25742574257425743,
       "cc": -31.644341476391443,
       "cc_by_type": {
        "choice": -7.317073170731707,
        "noul": -18.518518518518523,
        "score": -69.0974327399241
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.3541666666666667,
       "cc": -5.944000384605097,
       "cc_by_type": {
        "choice": 4.761904761904761,
        "noul": -60.0,
        "score": 37.40609408427995
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0.0
      },
      "jaggedness": {
       "accuracy": 0.37222222222222223,
       "cc": -8.884037970016676,
       "cc_by_type": {
        "choice": 24.927815206929747,
        "noul": -45.945945945945944,
        "score": -5.6339831710338295
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.23003194888178913,
       "cc": -20.437719287542667,
       "cc_by_type": {
        "choice": -0.6427753808223877,
        "noul": -38.144329896907216,
        "score": -22.526052584898395
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0.0
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.18518518518518517,
       "cc": -32.03827054080279,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75.0,
        "score": -22.145739457459925
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.29454545454545455,
       "cc": -25.78754754021827,
       "cc_by_type": {
        "choice": 17.835017932833402,
        "noul": -55.55555555555556,
        "score": -39.64210499793266
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -37.503184879754805,
       "cc_by_type": {
        "choice": 6.666666666666665,
        "noul": -13.33333333333333,
        "score": -105.84288797259775
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.20833333333333334,
       "cc": -1.1984021304926722,
       "cc_by_type": {
        "choice": -1.1984021304926722
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 0.0
      },
      "perturb_pair": {
       "accuracy": 0.265625,
       "cc": -25.52938809242961,
       "cc_by_type": {
        "choice": -5.105105105105101,
        "noul": 19.999999999999996,
        "score": -91.48305917218373
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0.0
      },
      "usecase": {
       "accuracy": 0.2409090909090909,
       "cc": -16.515408619322642,
       "cc_by_type": {
        "choice": 3.1862745098039587,
        "noul": -25.0,
        "score": -27.732500367771884
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -2.776462098911864,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -51.186529153878446
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.14285714285714285,
       "cc": 36.84210526315789,
       "cc_by_type": {
        "choice": -26.315789473684216,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 36.84210526315789
      },
      "da": {
       "accuracy": 0.3076923076923077,
       "cc": -37.718345339022456,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -100.0,
        "score": -50.9430544502471
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.4375,
       "cc": 22.354227161079496,
       "cc_by_type": {
        "choice": 20.353982300884955,
        "noul": 25.0,
        "score": 21.70869918235354
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 22.354227161079496
      },
      "el": {
       "accuracy": 0.2,
       "cc": 0.6000833683740154,
       "cc_by_type": {
        "choice": -17.31843575418994,
        "noul": -33.333333333333336,
        "score": 52.452019192645324
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.6000833683740154
      },
      "en": {
       "accuracy": 0.2797567332754127,
       "cc": -16.97684900279254,
       "cc_by_type": {
        "choice": 3.7844302607973472,
        "noul": -26.618705035971225,
        "score": -28.096272233203745
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.41935483870967744,
       "cc": -11.423096652088782,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -25.0,
        "score": -40.75379240324841
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -39.640298601530766,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": -20.56024006688737
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.21428571428571427,
       "cc": -41.1336147354147,
       "cc_by_type": {
        "choice": 15.294117647058824,
        "noul": -100.0,
        "score": -38.69496185330294
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.14285714285714285,
       "cc": -67.07248359384337,
       "cc_by_type": {
        "choice": -0.5586592178770929,
        "noul": -100.0,
        "score": -100.65879156365303
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.3333333333333333,
       "cc": -25.47727987845241,
       "cc_by_type": {
        "choice": 15.822291504286824,
        "noul": -33.333333333333336,
        "score": -58.92079780631072
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ja": {
       "accuracy": 0.24,
       "cc": -18.87257764229911,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -80.0,
        "score": -5.132427128644745
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.0,
       "cc": -77.83191986347525,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -100.16242625709242
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.35714285714285715,
       "cc": -10.156143207655505,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -33.333333333333336,
        "score": -13.801762956299847
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "nl": {
       "accuracy": 0.2,
       "cc": -51.11558643410791,
       "cc_by_type": {
        "choice": -29.87012987012987,
        "noul": -33.333333333333336,
        "score": -90.14329609886053
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.23076923076923078,
       "cc": -31.24251604325812,
       "cc_by_type": {
        "choice": 16.208209902666106,
        "noul": -80.0,
        "score": -29.93575803244046
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.39285714285714285,
       "cc": 2.5150561249822148,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 19.999999999999996,
        "score": -38.20615749423426
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 2.5150561249822148
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -37.55166369327866,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100.0,
        "score": -10.52733150536791
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.38461538461538464,
       "cc": -27.04550827576324,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100.0,
        "score": -15.919133522941898
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -41.81990641185333,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100.0,
        "score": -28.68552568717291
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -68.47760196419559,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100.0,
        "score": -129.1616194519088
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.272,
       "cc": 5.968349949291043,
       "cc_by_type": {
        "choice": 5.968349949291043
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 5.968349949291043
      },
      "noul": {
       "accuracy": 0.328,
       "cc": -34.4,
       "cc_by_type": {
        "noul": -34.4
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.24533333333333332,
       "cc": -29.344915811166672,
       "cc_by_type": {
        "score": -29.344915811166672
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -38.4024160564452,
       "cc_by_type": {
        "choice": -12.0,
        "noul": -75.0,
        "score": -28.207248169335596
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -2.238256630096153,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -11.111111111111116,
        "score": -39.70303766116492
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.42857142857142855,
       "cc": -3.7905034661254127,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 50.0,
        "score": -80.07069739024615
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -50.406556307064626,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -69.34832389195411
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -8.733832465640946,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -33.333333333333336,
        "score": -20.40439594764747
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -52.37905037945314,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": 0.0,
        "score": -169.08683667295062
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.23333333333333334,
       "cc": -29.56253397544977,
       "cc_by_type": {
        "choice": 21.568627450980394,
        "noul": -71.42857142857143,
        "score": -38.827657948758265
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.09090909090909091,
       "cc": -54.303624656078085,
       "cc_by_type": {
        "choice": -4.529616724738673,
        "noul": -100.0,
        "score": -58.38125724349559
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.06666666666666667,
       "cc": -27.932241245491838,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100.0,
        "score": 20.369942930191154
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -4.197964663778406,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 0.0,
        "score": -9.810168081270977
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.24,
       "cc": -7.467987900190109,
       "cc_by_type": {
        "choice": 6.976744186046504,
        "noul": -33.333333333333336,
        "score": 3.9526254467165067
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -26.741444679738834,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -17.858742641367044
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.2962962962962963,
       "cc": -17.699209031887154,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -60.0,
        "score": -12.483423640767022
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.22580645161290322,
       "cc": -17.59559356925139,
       "cc_by_type": {
        "choice": -5.194805194805169,
        "noul": -33.333333333333336,
        "score": -14.258642179615656
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0.0
      },
      "moderation_trust_safety": {
       "accuracy": 0.16666666666666666,
       "cc": -51.88151472980022,
       "cc_by_type": {
        "choice": -3.646833013435703,
        "noul": -100.0,
        "score": -51.99771117596499
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.4444444444444444,
       "cc": 0.4935502236767717,
       "cc_by_type": {
        "choice": -0.2785515320334252,
        "noul": 33.33333333333333,
        "score": -31.57413113026959
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.4935502236767717
      },
      "risk_assessment": {
       "accuracy": 0.1111111111111111,
       "cc": -49.23270679758713,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100.0,
        "score": -29.471519407539716
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.08695652173913043,
       "cc": -38.04052057494357,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -66.66666666666667,
        "score": -31.5128660726568
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.20833333333333334,
       "cc": -9.065556816294313,
       "cc_by_type": {
        "choice": 3.914590747330961,
        "noul": 0.0,
        "score": -31.111261196213903
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      13.378936392289322,
      21.888965168274606
     ],
     "I": [
      0.7771519016149122,
      4.933627097063504
     ],
     "capability": [
      7.593761103357901,
      12.813165041600257
     ],
     "composite": {
      "A": [
       0.0007030229261073844,
       0.1394909016822168
      ],
      "B": [
       0.0004538884390108904,
       0.10093093032471417
      ],
      "C": [
       0.0004882103653523503,
       0.09686868172376169
      ]
     },
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 17.26803648238681,
     "capability": 9.854654871580568,
     "intelligence": 2.4412732607743255
    },
    "per_type": {
     "choice": {
      "accuracy": 0.272,
      "calibration": 1.67473081339361,
      "cc": 5.968349949291043,
      "cc_by_type": {
       "choice": 5.968349949291043
      },
      "cc_tiered": 7.3238197823229765,
      "cc_tiered_open": 8.982230802737568,
      "cc_tiered_sealed": 5.6654087619083855,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 5.968349949291043
     },
     "noul": {
      "accuracy": 0.328,
      "calibration": 13.545463846169902,
      "cc": -34.4,
      "cc_by_type": {
       "noul": -34.4
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.24533333333333332,
      "calibration": 36.583914787596925,
      "cc": -29.344915811166672,
      "cc_by_type": {
       "score": -29.344915811166672
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 19.70003977436594,
     "capability": 11.004198815490696,
     "intelligence": 2.3083578566154506
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.016492932352696162,
    "Emphasis on Accuracy 60:20:20": 0.00951384924652731,
    "Emphasis on Cost 20:20:60": 0.026144624889269167,
    "Emphasis on Speed 20:60:20": 0.02594882609560486,
    "Intelligence only": 0.005819814930094012
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Axel Montzamir (Okura66)",
   "axes": {
    "calibration": 83.76747077065221,
    "cost": 50.12545250055582,
    "intelligence": 39.71971801341685,
    "speed": 89.23426082115222
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06004736643320625,
      "ece_score": 87.99052671335875,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.99052671335875,
      "score_before_time_limit": 87.99052671335875,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2991913746630728,
      "brier": 0.1447023593915565,
      "ece": 0.10598766007071211,
      "n": 371,
      "score": 78.80246798585758,
      "score_before_time_limit": 78.80246798585758,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9214092140921409,
      "interval90_width": 0.3678861788617887,
      "n": 369,
      "nrps": 0.09917884531883965,
      "score": 84.50941761274032,
      "score_before_time_limit": 84.50941761274032,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10531640121317691
     }
    },
    "score": 83.76747077065221
   },
   "capability": 61.74359439203453,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     24.508895373814624,
     46.749200339544835
    ],
    "B": [
     21.78693558252924,
     43.45052825108324
    ],
    "C": [
     17.020066231815708,
     32.46472245801724
    ]
   },
   "cost": {
    "basis": "ESTIMATE, ours (the author's server returns no token usage; Florian 7 Oct 2026: use our own estimate): 829,523 input tokens counted with the author's tokenizer and prompt builder over the 1,624 v1.5 items x n_permutations 3 (the author's server default) = 2,488,569 tokens / 1,624 decisions x USD 0.03 per 1M input and USD 0.15 per 1M output (deepinfra:Qwen/Qwen3.5-4B, frozen 25 Sep 2026 snapshot), 0 output tokens = USD 0.045971 per 1,000 decisions; carried like every Qwen3.5-4B row on this board. Cross-check: the same count on the v1.6 common cost basis (1,477 items, x3) gives USD 0.0448 per 1,000 (within 3 %). Corrected in v1.6.37, no new measurement: the earlier cost of this row used USD 0.04 per 1M input; the frozen reference for Qwen/Qwen3.5-4B is USD 0.03 per 1M input and USD 0.15 per 1M output (DeepInfra, 25 Sep 2026 snapshot).",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04597110221674877
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Kahn1 4B (Okura66, Qwen3.5-4B LoRA merge, author's sysone engine)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-07",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX 5090, the author's own sysone engine (github.com/Okura66/kahn1) on vLLM 0.30.0 with the author's documented max-model-len 4096 and calibration.json, offline (all credentials unset), driven over loopback only by the JevBench client through a thin pre-registered adapter for the author's /v1/evaluate/jev endpoint; label-free inputs; pod destroyed afterwards; JevBench add-request, measured 7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 41.96168150915031,
    "I_sealed": 37.4777545176834,
    "base": 39.71971801341685,
    "excess": 1.910539349028665,
    "gap": 4.483926991466909,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 62.694715037494376,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 69.80182447310474,
       "judge": 52.537903757415954,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 17.904071773636996,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 45.28625771631957,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.624702928143726,
       "hard": 52.08153129449285,
       "judge": 35.187676087499,
       "standard": 44.67436039729178
      }
     },
     "sealed|choice": {
      "cc": 60.13243945421864,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 56.730769230769226,
       "judge": 57.16945996275605,
       "standard": 66.7446729892844
      }
     },
     "sealed|noul": {
      "cc": 16.581379950628655,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -10.679611650485432,
       "judge": 28.947368421052634,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 35.719444148202925,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 55.76848705914599,
       "hard": 35.73652095498605,
       "judge": 30.714342076628732,
       "standard": 33.16842218652643
      }
     }
    }
   },
   "jevbench_score": 36.97416955749114,
   "key": "kahn1",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Okura66/Kahn1-Qwen3.5-4B@f3f9910f995b0e345a6e8a9cb0aeed1e06bafaf6",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8653846153846154,
    "decisive_rate": 0.7008086253369272,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.76747077065221,
     "cost": 50.12545250055582,
     "intelligence": 39.71971801341685,
     "speed": 89.97531755002697
    },
    "board_release": "v1.6.43",
    "capability": 61.74359439203453,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 54,
    "capability_rank": 79,
    "composite_ci95": {
     "A": [
      24.53974660316557,
      46.8147086789254
     ],
     "B": [
      21.806431754904573,
      43.49585293740054
     ],
     "C": [
      17.041490696642754,
      32.51021436036486
     ]
    },
    "jevbench_score": 37.0242246815327,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 59,
    "rank": 89,
    "ranks": {
     "A": 89,
     "B": 87,
     "C": 90,
     "capability": 123
    },
    "scores": {
     "A": 37.0242246815327,
     "B": 33.799163379528274,
     "C": 25.711267139953264
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2821926829987206,
     "p50_s_raw": 0.06609634149936028,
     "p95_s_adjusted": 0.3563875272346194,
     "p95_s_raw": 0.10319376361730971
    },
    "views": {
     "Balanced 33:33:33": 33.66233685744503,
     "Emphasis on Accuracy 60:20:20": 29.601398852667728,
     "Emphasis on Cost 20:20:60": 32.81983249388412,
     "Emphasis on Speed 20:60:20": 40.21102161322423,
     "Intelligence only": 25.065620562020474
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a8 (scorer output sha256 7a5b4d4c7b7c, method option B / O1S)"
   },
   "rank": 101,
   "ranked": true,
   "ranks": {
    "A": 101,
    "B": 99,
    "C": 103,
    "capability": 139
   },
   "repo": "https://huggingface.co/Okura66/Kahn1-Qwen3.5-4B",
   "repriced_in": "v1.6.37",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 36.97416955749114,
    "B": 33.765779495876835,
    "C": 25.676506637146623
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.30520028799073773,
    "p50_s_raw": 0.07760014399536885,
    "p95_s_adjusted": 0.3908309774473309,
    "p95_s_raw": 0.12041548872366546
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "supersedes_base_row": "v1.6.1",
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.07087093913225,
      85.02256483809754
     ],
     "I": [
      33.5350460501225,
      43.90853133816846
     ],
     "capability": [
      56.37025407029597,
      63.949534098181175
     ],
     "composite": {
      "A": [
       24.508895373814624,
       46.749200339544835
      ],
      "B": [
       21.78693558252924,
       43.45052825108324
      ],
      "C": [
       17.020066231815708,
       32.46472245801724
      ]
     },
     "seed": 60
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.76747077065221,
     "capability": 61.74359439203453,
     "intelligence": 39.71971801341685
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7026666666666667,
      "calibration": 87.99052671335875,
      "cc": 61.97531332000794,
      "cc_by_type": {
       "choice": 61.97531332000794
      },
      "cc_tiered": 61.41357724585651,
      "cc_tiered_open": 62.694715037494376,
      "cc_tiered_sealed": 60.13243945421864,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.97531332000794
     },
     "noul": {
      "accuracy": 0.6,
      "calibration": 78.80246798585758,
      "cc": 19.999999999999996,
      "cc_by_type": {
       "noul": 19.999999999999996
      },
      "cc_tiered": 17.242725862132826,
      "cc_tiered_open": 17.904071773636996,
      "cc_tiered_sealed": 16.581379950628655,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 19.999999999999996
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 84.50941761274032,
      "cc": 39.16844975336833,
      "cc_by_type": {
       "score": 39.16844975336833
      },
      "cc_tiered": 40.502850932261246,
      "cc_tiered_open": 45.28625771631957,
      "cc_tiered_sealed": 35.719444148202925,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.16844975336833
     }
    },
    "raw_on_A": {
     "calibration": 81.01531873303975,
     "capability": 63.27790509971841,
     "intelligence": 45.54049146639707
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 33.6071825743108,
    "Emphasis on Accuracy 60:20:20": 29.575789257966896,
    "Emphasis on Cost 20:20:60": 32.788354278198014,
    "Emphasis on Speed 20:60:20": 40.06962605109233,
    "Intelligence only": 25.065620562020474
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Sqwish",
   "axes": {
    "calibration": 84.10309591585386,
    "cost": 62.69047197239138,
    "intelligence": 62.182714942969874,
    "speed": 94.89692360127322
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09316854395599922,
      "ece_score": 81.36629120880016,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 81.36629120880016,
      "score_before_time_limit": 81.36629120880016,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.13404825737265413,
      "brier": 0.09010097510472304,
      "ece": 0.0736929319861733,
      "n": 373,
      "score": 85.26141360276533,
      "score_before_time_limit": 85.26141360276533,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8736559139784946,
      "interval90_width": 0.24484767025089602,
      "n": 372,
      "nrps": 0.08489712327077574,
      "score": 85.68158293599608,
      "score_before_time_limit": 85.68158293599608,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10073560900465137
     }
    },
    "score": 84.10309591585386
   },
   "capability": 73.14290542941187,
   "composite_ci95": {
    "A": [
     70.52362840431199,
     74.9752910408981
    ],
    "B": [
     67.09508922559976,
     73.08094665903728
    ],
    "C": [
     61.9510288460444,
     74.9752910408981
    ]
   },
   "cost": {
    "basis": "Sqwish public price list, USD 0.04/M input, output not billed (docs.sqwish.ai/guides/pricing, fetched 16 Oct 2026 16:17Z, saved copy sha256 698a29e5...): kettei chub at USD 0.04 per million input tokens for decisions; \"The models generate no output tokens, and none are billed\"; \"standard list prices, the same since launch on 14 October 2026\". Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4 and Nimble), through a scoped re-freeze of the frozen pricing module that adds the two kettei keys and nothing else, with a zero-difference proof for every other registry entry. The once-per-inbox USD 5.00 starter credit is a trial credit and is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (650,605 input tokens over the 1,485 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis the scorer uses; the 15 items refused with HTTP 422 count as zero tokens and as wrong answers, as every refused request on this draw does (d1-ultra's identical 15 refusals included). The unchanged runner also wrote a conservative cost ceiling into those 15 refused rows (65,536 input tokens each, the model's input limit); the scoring copy drops that one field on those 15 rows only. With the ceiling charged on all 1,500 items (the former all-items basis), the figure would be USD 0.043564 per 1,000 decisions (0.674x the official cost cap); that is a sensitivity, not this row's cost, and on the common basis the 15 refused items are not part of cost at all. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.017525 per 1,000 decisions (0.271x the official cost cap USD 0.064595), against USD 0.017349 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.017349466666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.017524713804713803
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kettei-chub (Sqwish)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.sqwish.ai/v1/systemone API, model `kettei-chub` (Sqwish docs: kettei-chub v1.0, released 14 Oct 2026), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `sqwish` before the first request (custody #20263, deputy #20305/#20306). Native probabilities on all three question types. The runner recorded kettei-chub as the answering model on all 1,500 rows (the adapter reads the response's model field), so no row was answered by a fallback size. 15 of 1,500 items exceed the model's documented 65,536-token input limit (HTTP 422 context_too_long) and are scored as failures. Paid public fast-lane order 6e10efe6, run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 65.83625821719068,
    "I_sealed": 58.52917166874907,
    "base": 62.182714942969874,
    "excess": -2.532440221914108,
    "gap": 7.307086548441603,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.058241050527,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 62.25228059138094,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 71.38647342995169,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 70.0,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 52.064060171093324,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.46376557960574,
       "hard": 63.16197369506109,
       "judge": 50.34751163838864,
       "standard": 28.74320321795858
      }
     },
     "sealed|choice": {
      "cc": 74.30006625391061,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 88.56416772554002,
       "hard": 69.81612119810339,
       "judge": 69.14789422135163,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 54.66369142951747,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 41.81818181818182,
       "judge": 56.92307692307692,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 46.62375732281914,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 75.69949020400291,
       "hard": 37.65779302682666,
       "judge": 46.53750160134448,
       "standard": 50.147203056424175
      }
     }
    }
   },
   "jevbench_score": 73.44712821007478,
   "key": "kettei-chub",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.sqwish.ai/v1/systemone#kettei-chub",
   "noul_decisive": {
    "acc_among_decisive": 0.9380804953560371,
    "decisive": 323,
    "decisive_rate": 0.8659517426273459,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 9,
    "axes": {
     "calibration": 84.10309591585386,
     "cost": 62.69047197239138,
     "intelligence": 62.182714942969874,
     "speed": 95.09167642607092
    },
    "board_release": "v1.6.43",
    "capability": 73.14290542941187,
    "capability_api_board_rank": 16,
    "capability_open_board_rank": null,
    "capability_rank": 32,
    "composite_ci95": {
     "A": [
      70.55047342358355,
      75.00563282943165
     ],
     "B": [
      67.11452609813675,
      73.104006933403
     ],
     "C": [
      61.974600790042366,
      75.00563282943165
     ]
    },
    "jevbench_score": 73.47624549745721,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 11,
    "ranks": {
     "A": 11,
     "B": 16,
     "C": 11,
     "capability": 57
    },
    "scores": {
     "A": 73.47624549745721,
     "B": 70.90086398046398,
     "C": 73.47624549745721
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.16667446299106814,
     "p50_s_raw": 0.16667446299106814,
     "p95_s_adjusted": 0.18576474376022814,
     "p95_s_raw": 0.18576474376022814
    },
    "views": {
     "Balanced 33:33:33": 70.50662300412756,
     "Emphasis on Accuracy 60:20:20": 66.92322969828575,
     "Emphasis on Cost 20:20:60": 67.15739116260939,
     "Emphasis on Speed 20:60:20": 78.6391905348317,
     "Intelligence only": 62.182714942969874
    }
   },
   "rank": 13,
   "ranked": true,
   "ranks": {
    "A": 13,
    "B": 18,
    "C": 13,
    "capability": 68
   },
   "repo": "https://docs.sqwish.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.44712821007478,
    "B": 70.8791724952149,
    "C": 73.44712821007478
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.16808052101987414,
    "p50_s_raw": 0.16808052101987414,
    "p95_s_adjusted": 0.19265942253987306,
    "p95_s_raw": 0.19265942253987306
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 63.43823636827809,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 6.981375771500953
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.43823636827809
      },
      "calibration_gen": {
       "accuracy": 0.801980198019802,
       "cc": 63.075183513321825,
       "cc_by_type": {
        "choice": 87.27465535524921,
        "noul": 60.00000000000001,
        "score": 41.950895184716266
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 63.075183513321825
      },
      "composite_gen": {
       "accuracy": 0.9702970297029703,
       "cc": 94.94824383168736,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 100.0,
        "score": 88.09676401538727
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 94.94824383168736
      },
      "fairness": {
       "accuracy": 0.9166666666666666,
       "cc": 79.68499498178473,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 46.66666666666666,
        "score": 92.38831827868752
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 79.68499498178473
      },
      "jaggedness": {
       "accuracy": 0.5888888888888889,
       "cc": 24.7828907621076,
       "cc_by_type": {
        "choice": 74.01347449470644,
        "noul": 8.108108108108114,
        "score": -7.772910316491766
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 24.7828907621076
      },
      "llm_router": {
       "accuracy": 0.8146964856230032,
       "cc": 71.82742478150287,
       "cc_by_type": {
        "choice": 68.3014880690323,
        "noul": 83.50515463917525,
        "score": 63.675631636301034
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.82742478150287
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 23.877597944583083,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": 25.0,
        "score": 20.859597957460597
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 23.877597944583083
      },
      "multilingual": {
       "accuracy": 0.7709090909090909,
       "cc": 63.90048221318355,
       "cc_by_type": {
        "choice": 74.5679817411151,
        "noul": 61.111111111111114,
        "score": 56.02235378732444
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.90048221318355
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 72.10157892455614,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 53.33333333333334,
        "score": 69.63807010700175
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 72.10157892455614
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.921875,
       "cc": 86.78313259496274,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 72.36140979690022
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.78313259496274
      },
      "usecase": {
       "accuracy": 0.6818181818181818,
       "cc": 55.999296711189196,
       "cc_by_type": {
        "choice": 64.4607843137255,
        "noul": 50.0,
        "score": 53.53710581984207
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 55.999296711189196
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 28.24640647098315,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -8.117923444193398
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 28.24640647098315
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 71.36869026296806,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 100.0,
        "score": 55.58072516678434
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 71.36869026296806
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.2987215870243,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100.0,
        "score": 81.7545718407189
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.2987215870243
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 71.77688256637983,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 66.66666666666667,
        "score": 98.94331064141133
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 71.77688256637983
      },
      "en": {
       "accuracy": 0.7732406602953953,
       "cc": 61.55931282182781,
       "cc_by_type": {
        "choice": 75.39693193883403,
        "noul": 61.15107913669065,
        "score": 48.12992738995876
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 61.55931282182781
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 76.64478118166096,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 39.722271766842574
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 76.64478118166096
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.73563989131308,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 96.66047158650755
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.73563989131308
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 79.01802864795974,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 79.40702712034981
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 79.01802864795974
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 24.206837327473764,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 50.0,
        "score": -27.10015840864015
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 24.206837327473764
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 76.75527283143386,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 44.295436576920444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 76.75527283143386
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 48.64168346881549,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 60.00000000000001,
        "score": 0.22211156609699412
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 48.64168346881549
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 49.09081184412445,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 47.27243553237336
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 49.09081184412445
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 98.50661742898735,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.51985228696206
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.50661742898735
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 60.007042228304385,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 98.63584529963177
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 60.007042228304385
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 61.31631791133893,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 51.566389197410786
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.31631791133893
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 63.091184704198746,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 61.09441445437582
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 63.091184704198746
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 54.828846215251076,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 55.55036843298726
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 54.828846215251076
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 56.63034829618355,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": -8.369824676666738
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 56.63034829618355
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 82.98189368154357,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 81.20374556075973
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 82.98189368154357
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 48.085566824177704,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 10.92336713919978
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.085566824177704
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8066666666666666,
       "cc": 75.02822480338315,
       "cc_by_type": {
        "choice": 75.02822480338315
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.02822480338315
      },
      "noul": {
       "accuracy": 0.808,
       "cc": 61.60000000000001,
       "cc_by_type": {
        "noul": 61.60000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 61.60000000000001
      },
      "score": {
       "accuracy": 0.672,
       "cc": 49.184305491330974,
       "cc_by_type": {
        "score": 49.184305491330974
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 49.184305491330974
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 32.94973949397677,
       "cc_by_type": {
        "choice": 36.0,
        "noul": 25.0,
        "score": 37.8492184819303
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 32.94973949397677
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 21.86713652267862,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 33.33333333333333,
        "score": -11.831302647285046
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 21.86713652267862
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 58.41468804345843,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 7.764389333627342
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 58.41468804345843
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 21.307121699711207,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": 24.155283227788605
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 21.307121699711207
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 52.732202808738386,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 53.84878233925865
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 52.732202808738386
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": -17.90003745614136,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 66.66666666666667,
        "score": -157.47369727408446
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 50.15463162953395,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 37.018516737341336
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 50.15463162953395
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 46.19866186338035,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 0.0,
        "score": 80.40783228003652
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 46.19866186338035
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 55.891544536440826,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 70.1746336093225
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 55.891544536440826
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 61.85269108394834,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 44.58733806269443
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 61.85269108394834
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 72.34106793586746,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 50.87333300656878
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 72.34106793586746
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 40.88339357227535,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 33.33333333333333,
        "score": 21.574911899621764
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 40.88339357227535
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 51.77378065698675,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 58.3539715295015
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 51.77378065698675
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 78.0080271816572,
       "cc_by_type": {
        "choice": 84.80519480519482,
        "noul": 96.96969696969697,
        "score": 52.24918977007981
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 78.0080271816572
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 41.95050174639673,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0.0,
        "score": 60.40044957700209
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 41.95050174639673
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 56.20139697883294,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 68.69704144717662
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.20139697883294
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 36.63384674487239,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 50.0,
        "score": 19.014840727228023
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 36.63384674487239
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 40.942225675125954,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 66.30493789494308
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 40.942225675125954
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.41000709626154,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 93.90617787241452
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.41000709626154
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.03674454380982,
      86.11285311974031
     ],
     "I": [
      56.070327300946786,
      66.51811397342259
     ],
     "capability": [
      68.26370889954131,
      75.68024782241592
     ],
     "composite": {
      "A": [
       70.52362840431199,
       74.9752910408981
      ],
      "B": [
       67.09508922559976,
       73.08094665903728
      ],
      "C": [
       61.9510288460444,
       74.9752910408981
      ]
     },
     "seed": 40
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.10309591585386,
     "capability": 73.14290542941187,
     "intelligence": 62.182714942969874
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8066666666666666,
      "calibration": 81.36629120880016,
      "cc": 75.02822480338315,
      "cc_by_type": {
       "choice": 75.02822480338315
      },
      "cc_tiered": 74.17915365221882,
      "cc_tiered_open": 74.058241050527,
      "cc_tiered_sealed": 74.30006625391061,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.02822480338315
     },
     "noul": {
      "accuracy": 0.808,
      "calibration": 85.26141360276533,
      "cc": 61.60000000000001,
      "cc_by_type": {
       "noul": 61.60000000000001
      },
      "cc_tiered": 63.02508242973458,
      "cc_tiered_open": 71.38647342995169,
      "cc_tiered_sealed": 54.66369142951747,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 61.60000000000001
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 85.68158293599608,
      "cc": 49.184305491330974,
      "cc_by_type": {
       "score": 49.184305491330974
      },
      "cc_tiered": 49.34390874695623,
      "cc_tiered_open": 52.064060171093324,
      "cc_tiered_sealed": 46.62375732281914,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 49.184305491330974
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.47087864836172,
    "Emphasis on Accuracy 60:20:20": 66.90390345364955,
    "Emphasis on Cost 20:20:60": 67.1379294576571,
    "Emphasis on Speed 20:60:20": 78.55919303268641,
    "Intelligence only": 62.182714942969874
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Sqwish",
   "axes": {
    "calibration": 92.78078361709031,
    "cost": 57.40773420072094,
    "intelligence": 79.83918785475791,
    "speed": 92.90799647527135
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.024857318118089264,
      "ece_score": 95.02853637638215,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 95.02853637638215,
      "score_before_time_limit": 95.02853637638215,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.04557640750670244,
      "brier": 0.04418057925447822,
      "ece": 0.03819155325961758,
      "n": 373,
      "score": 92.36168934807648,
      "score_before_time_limit": 92.36168934807648,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9301075268817204,
      "interval90_width": 0.1270161290322581,
      "n": 372,
      "nrps": 0.044540233632597244,
      "score": 90.95212512681232,
      "score_before_time_limit": 90.95212512681232,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0682086319155783
     }
    },
    "score": 92.78078361709031
   },
   "capability": 86.3099857359241,
   "composite_ci95": {
    "A": [
     76.06572376300029,
     78.39596115963143
    ],
    "B": [
     75.94727293323967,
     79.2430710140646
    ],
    "C": [
     76.06572376300029,
     78.39596115963143
    ]
   },
   "cost": {
    "basis": "Sqwish public price list, USD 0.06/M input, output not billed (docs.sqwish.ai/guides/pricing, fetched 16 Oct 2026 16:17Z, saved copy sha256 698a29e5...): kettei fat at USD 0.06 per million input tokens for decisions; \"The models generate no output tokens, and none are billed\"; \"standard list prices, the same since launch on 14 October 2026\". Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4 and Nimble), through a scoped re-freeze of the frozen pricing module that adds the two kettei keys and nothing else, with a zero-difference proof for every other registry entry. The once-per-inbox USD 5.00 starter credit is a trial credit and is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (650,605 input tokens over the 1,485 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis the scorer uses; the 15 items refused with HTTP 422 count as zero tokens and as wrong answers, as every refused request on this draw does (d1-ultra's identical 15 refusals included). The unchanged runner also wrote a conservative cost ceiling into those 15 refused rows (65,536 input tokens each, the model's input limit); the scoring copy drops that one field on those 15 rows only. With the ceiling charged on all 1,500 items (the former all-items basis), the figure would be USD 0.065346 per 1,000 decisions (1.012x the official cost cap); that is a sensitivity, not this row's cost, and on the common basis the 15 refused items are not part of cost at all. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.026287 per 1,000 decisions (0.407x the official cost cap USD 0.064595), against USD 0.026024 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.026024199999999994
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0262870707070707
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kettei-fat (Sqwish)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.sqwish.ai/v1/systemone API, model `kettei-fat` (Sqwish docs: kettei-fat v1.0, released 14 Oct 2026), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `sqwish` before the first request (custody #20263, deputy #20305/#20306). Native probabilities on all three question types. The runner recorded kettei-fat as the answering model on all 1,500 rows (the adapter reads the response's model field), so no row was answered by a fallback size. 15 of 1,500 items exceed the model's documented 65,536-token input limit (HTTP 422 context_too_long) and are scored as failures. Paid public fast-lane order a87f712c, run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.51365708398285,
    "I_sealed": 80.16471862553297,
    "base": 79.83918785475791,
    "excess": -10.490588311905832,
    "gap": -0.6510615415501206,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 87.36347037521236,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 86.78829820698331,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 80.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 71.10020619074588,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 82.05684261096903,
       "hard": 73.02709979951186,
       "judge": 72.7279902420528,
       "standard": 59.32642468614199
      }
     },
     "sealed|choice": {
      "cc": 87.64254301487159,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 100.0,
       "hard": 81.78559037816582,
       "judge": 89.42213516160628,
       "standard": 90.50833157561695
      }
     },
     "sealed|noul": {
      "cc": 83.10326832200224,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 93.10344827586208,
       "hard": 74.54545454545453,
       "judge": 93.84615384615384,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 69.74834453972508,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 89.41045649485207,
       "hard": 62.86673539910686,
       "judge": 69.73247418036178,
       "standard": 73.70431238244299
      }
     }
    }
   },
   "jevbench_score": 77.69122909321317,
   "key": "kettei-fat",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.sqwish.ai/v1/systemone#kettei-fat",
   "noul_decisive": {
    "acc_among_decisive": 0.9634831460674157,
    "decisive": 356,
    "decisive_rate": 0.9544235924932976,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 1,
    "axes": {
     "calibration": 92.78078361709031,
     "cost": 57.40773420072094,
     "intelligence": 79.83918785475791,
     "speed": 93.19124027169539
    },
    "board_release": "v1.6.43",
    "capability": 86.3099857359241,
    "capability_api_board_rank": 1,
    "capability_open_board_rank": null,
    "capability_rank": 1,
    "composite_ci95": {
     "A": [
      76.11307381850703,
      78.44625770405383
     ],
     "B": [
      75.98503036462549,
      79.28417747037292
     ],
     "C": [
      76.11307381850703,
      78.44625770405383
     ]
    },
    "jevbench_score": 77.74062514641476,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 1,
    "ranks": {
     "A": 1,
     "B": 1,
     "C": 1,
     "capability": 6
    },
    "scores": {
     "A": 77.74062514641476,
     "B": 78.15146537474011,
     "C": 77.74062514641476
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.20005768549162894,
     "p50_s_raw": 0.20005768549162894,
     "p95_s_adjusted": 0.2397290877619525,
     "p95_s_raw": 0.2397290877619525
    },
    "views": {
     "Balanced 33:33:33": 73.7552774018824,
     "Emphasis on Accuracy 60:20:20": 76.0740779326146,
     "Emphasis on Cost 20:20:60": 66.21326749074146,
     "Emphasis on Speed 20:60:20": 80.46826083335421,
     "Intelligence only": 79.83918785475791
    }
   },
   "rank": 2,
   "ranked": true,
   "ranks": {
    "A": 2,
    "B": 2,
    "C": 2,
    "capability": 5
   },
   "repo": "https://docs.sqwish.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 77.69122909321317,
    "B": 78.11152478647084,
    "C": 77.69122909321317
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20237550800084136,
    "p50_s_raw": 0.20237550800084136,
    "p95_s_adjusted": 0.2529544946068199,
    "p95_s_raw": 0.2529544946068199
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 96.7507472305415,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 98.58557502495783
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 96.7507472305415
      },
      "calibration_gen": {
       "accuracy": 0.9603960396039604,
       "cc": 93.47881080044199,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92.0,
        "score": 91.6177685625137
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 93.47881080044199
      },
      "composite_gen": {
       "accuracy": 0.9900990099009901,
       "cc": 98.5804818840001,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.74144565200028
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 98.5804818840001
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 84.86062527550072,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 94.58187582650217
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.86062527550072
      },
      "jaggedness": {
       "accuracy": 0.6777777777777778,
       "cc": 48.399546734792864,
       "cc_by_type": {
        "choice": 74.01347449470644,
        "noul": 35.13513513513513,
        "score": 36.05003057453702
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 48.399546734792864
      },
      "llm_router": {
       "accuracy": 0.952076677316294,
       "cc": 87.82993917915225,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 97.93814432989691,
        "score": 68.72152440065662
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 87.82993917915225
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.8518518518518519,
       "cc": 83.41157557420702,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 100.0,
        "score": 87.34812878447671
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 83.41157557420702
      },
      "multilingual": {
       "accuracy": 0.8618181818181818,
       "cc": 78.9798737677216,
       "cc_by_type": {
        "choice": 80.43690903162701,
        "noul": 86.11111111111111,
        "score": 70.39160116042665
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 78.9798737677216
      },
      "negation_pair_gen": {
       "accuracy": 0.9767441860465116,
       "cc": 94.85365625545666,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 86.66666666666667,
        "score": 97.8943020997033
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 94.85365625545666
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 89.64749433680471,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100.0,
        "score": 74.94848901642015
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 89.64749433680471
      },
      "usecase": {
       "accuracy": 0.8590909090909091,
       "cc": 79.73490795932031,
       "cc_by_type": {
        "choice": 88.97058823529412,
        "noul": 75.0,
        "score": 75.2341356426668
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 79.73490795932031
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 86.21024275740876,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 77.67834731984534
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 86.21024275740876
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 75.04719104747973,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 45.87890033137929
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 75.04719104747973
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.628058606648,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100.0,
        "score": 91.74258289959003
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.628058606648
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.61393066873585,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.69467841216657
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.61393066873585
      },
      "en": {
       "accuracy": 0.8835794960903562,
       "cc": 80.4883103640632,
       "cc_by_type": {
        "choice": 89.67549822433213,
        "noul": 80.57553956834533,
        "score": 71.21389329951214
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 80.4883103640632
      },
      "es": {
       "accuracy": 0.967741935483871,
       "cc": 90.39349165083537,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 71.18047495250612
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 90.39349165083537
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.5139405386418,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.2084882825921
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.5139405386418
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 91.9301217185329,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 96.966835743834
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 91.9301217185329
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 79.08766846078485,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100.0,
        "score": 54.022781918667405
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 79.08766846078485
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 80.52064370654271,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 55.59154920224702
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 80.52064370654271
      },
      "ja": {
       "accuracy": 0.92,
       "cc": 80.92306844140934,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 80.0,
        "score": 62.76920532422802
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 80.92306844140934
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 85.64585125280645,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 56.93755375841938
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 85.64585125280645
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 97.20181470806706,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 91.60544412420121
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 97.20181470806706
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 15.179397099416787,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": -61.821116061057005
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 15.179397099416787
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8717948717948718,
       "cc": 86.0234580413718,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 100.0,
        "score": 73.30524505090337
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 86.0234580413718
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 75.9847623403091,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 59.7751473627069
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.9847623403091
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 67.67066900776906,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 100.0,
        "score": 54.07583681054125
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 67.67066900776906
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 83.57430186853024,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 72.46203604037333
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 83.57430186853024
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 81.78317833096776,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 77.60759950903231
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 81.78317833096776
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 46.85913519178663,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 7.244072242026545
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.85913519178663
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.9053333333333333,
       "cc": 87.77244111062208,
       "cc_by_type": {
        "choice": 87.77244111062208
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 87.77244111062208
      },
      "noul": {
       "accuracy": 0.9146666666666666,
       "cc": 82.93333333333332,
       "cc_by_type": {
        "noul": 82.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 82.93333333333332
      },
      "score": {
       "accuracy": 0.7973333333333333,
       "cc": 70.94010320323034,
       "cc_by_type": {
        "score": 70.94010320323034
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 70.94010320323034
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 49.66614749506439,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 50.0,
        "score": 30.998442485193188
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 49.66614749506439
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 91.94328546901535,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 75.82985640704605
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 91.94328546901535
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 51.2790489623809,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": -13.642527909605295
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.2790489623809
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 14.651774389394646,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 100.0,
        "score": -74.1733318025763
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 14.651774389394646
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.9411764705882353,
       "cc": 90.47904341608493,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 71.43713024825477
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 90.47904341608493
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 47.918819771310574,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 66.66666666666667,
        "score": -10.331591000533692
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 47.918819771310574
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 77.48796585461322,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 71.42857142857142,
        "score": 61.03532613526828
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 77.48796585461322
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 66.73764536574002,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50.0,
        "score": 71.11885944216777
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 66.73764536574002
      },
      "insurance_claims": {
       "accuracy": 0.8666666666666667,
       "cc": 77.28653895219419,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 60.00000000000001,
        "score": 92.69295018991592
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 77.28653895219419
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 75.55654372603391,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 72.85093025019309
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 75.55654372603391
      },
      "lead_generation": {
       "accuracy": 0.88,
       "cc": 87.19694978627531,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 100.0,
        "score": 73.21875633557016
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 87.19694978627531
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 52.35634073814115,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 88.25181791334818
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 52.35634073814115
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 82.92727279464067,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 88.781818383922
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 82.92727279464067
      },
      "model_routing": {
       "accuracy": 0.9262672811059908,
       "cc": 84.3082332710354,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100.0,
        "score": 58.76885565726209
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 84.3082332710354
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 56.11137155389184,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0.0,
        "score": 79.85042944094617
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 56.11137155389184
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 89.83757082391922,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": 86.22580439376324
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 89.83757082391922
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 69.35928856610843,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100.0,
        "score": 37.63451594463072
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 69.35928856610843
      },
      "scientific_discovery": {
       "accuracy": 0.8695652173913043,
       "cc": 84.81707822400728,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 100.0,
        "score": 83.43674191839865
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 84.81707822400728
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.38110705803847,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.14332117411539
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.38110705803847
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.77958369550683,
      93.35538700668452
     ],
     "I": [
      75.36869158042848,
      83.13854440546274
     ],
     "capability": [
      82.60221375083708,
      87.8846855518411
     ],
     "composite": {
      "A": [
       76.06572376300029,
       78.39596115963143
      ],
      "B": [
       75.94727293323967,
       79.2430710140646
      ],
      "C": [
       76.06572376300029,
       78.39596115963143
      ]
     },
     "seed": 41
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.78078361709031,
     "capability": 86.3099857359241,
     "intelligence": 79.83918785475791
    },
    "per_type": {
     "choice": {
      "accuracy": 0.9053333333333333,
      "calibration": 95.02853637638215,
      "cc": 87.77244111062208,
      "cc_by_type": {
       "choice": 87.77244111062208
      },
      "cc_tiered": 87.50300669504198,
      "cc_tiered_open": 87.36347037521236,
      "cc_tiered_sealed": 87.64254301487159,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 87.77244111062208
     },
     "noul": {
      "accuracy": 0.9146666666666666,
      "calibration": 92.36168934807648,
      "cc": 82.93333333333332,
      "cc_by_type": {
       "noul": 82.93333333333332
      },
      "cc_tiered": 81.5902815039963,
      "cc_tiered_open": 80.07729468599035,
      "cc_tiered_sealed": 83.10326832200224,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 82.93333333333332
     },
     "score": {
      "accuracy": 0.7973333333333333,
      "calibration": 90.95212512681232,
      "cc": 70.94010320323034,
      "cc_by_type": {
       "score": 70.94010320323034
      },
      "cc_tiered": 70.42427536523547,
      "cc_tiered_open": 71.10020619074588,
      "cc_tiered_sealed": 69.74834453972508,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 70.94010320323034
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 73.69600561562211,
    "Emphasis on Accuracy 60:20:20": 76.03623197488143,
    "Emphasis on Cost 20:20:60": 66.184595083788,
    "Emphasis on Speed 20:60:20": 80.34136502055784,
    "Intelligence only": 79.83918785475791
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "jaredpalmer",
   "axes": {
    "calibration": 86.77132815303852,
    "cost": null,
    "intelligence": 36.22478886598313,
    "speed": 89.01765799382792
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0745927868852459,
      "ece_score": 85.08144262295082,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 85.08144262295082,
      "score_before_time_limit": 85.08144262295082,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.420911528150134,
      "brier": 0.1571680953083108,
      "ece": 0.07976219839142094,
      "n": 373,
      "score": 84.0475603217158,
      "score_before_time_limit": 84.0475603217158,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9704301075268817,
      "interval90_width": 0.5497311827956989,
      "n": 372,
      "nrps": 0.10286757401209681,
      "score": 91.18498151444894,
      "score_before_time_limit": 91.18498151444894,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0367163978494623
     }
    },
    "score": 86.77132815303852
   },
   "capability": 61.49805850951083,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base (the module RAISES 'base model lacks an explicit market-listing decision'); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's server, kept as sent (input 662,682; output 164,737 is the author's billing-style count of the serialised answers, not generated text).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Kev-9B v2 (Jared Palmer, Qwen3.5-9B-Base LoRA + pointer head)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's kev.serve on torch.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 44.923827258334626,
    "I_sealed": 27.52575047363164,
    "base": 36.22478886598313,
    "excess": 7.558550014347276,
    "gap": 17.398076784702987,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 63.253539541495684,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 62.25228059138094,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 24.727398205659078,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 46.79054402784911,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.67848484848486,
       "hard": 52.62685185185185,
       "judge": 44.33762711864405,
       "standard": 38.85333333333332
      }
     },
     "sealed|choice": {
      "cc": 55.791766173685986,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 71.41041931385007,
       "hard": 53.68335839019312,
       "judge": 52.39960822722822,
       "standard": 57.28749209027631
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -14.54545454545455,
       "judge": -32.30769230769231,
       "standard": 19.402985074626855
      }
     },
     "sealed|score": {
      "cc": 26.785485247208936,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 51.87780701754385,
       "hard": 19.02600000000001,
       "judge": 22.344166666666666,
       "standard": 36.42027272727272
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "kev-9b-v2",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "jaredpalmer/kev-9b@b5d8c18e44c60888d138b65cb6507ff0a5a448a0",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.8888888888888888,
    "decisive": 216,
    "decisive_rate": 0.579088471849866,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.77132815303852,
     "cost": null,
     "intelligence": 36.22478886598313,
     "speed": 89.1575041920417
    },
    "board_release": "v1.6.43",
    "capability": 61.49805850951083,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25193482227623465,
     "p50_s_raw": 0.05096741113811731,
     "p95_s_adjusted": 0.481904943287372,
     "p95_s_raw": 0.165952471643686
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/jaredpalmer/kev-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2587409914471209,
    "p50_s_raw": 0.05437049572356045,
    "p95_s_adjusted": 0.484583879681304,
    "p95_s_raw": 0.16729193984065202
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 56.42555555555557,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 2.6100000000000234
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 56.42555555555557
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 15.13716264581125,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": -12.0,
        "score": 17.856875000000016
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 15.13716264581125
      },
      "composite_gen": {
       "accuracy": 0.5445544554455446,
       "cc": 18.58461619530725,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -18.518518518518523,
        "score": 6.792692307692305
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 18.58461619530725
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 76.8387037037037,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 70.51611111111112
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 76.8387037037037
      },
      "jaggedness": {
       "accuracy": 0.39444444444444443,
       "cc": 1.3263832320119515,
       "cc_by_type": {
        "choice": 27.81520692974015,
        "noul": -51.35135135135135,
        "score": 27.51529411764705
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 1.3263832320119515
      },
      "llm_router": {
       "accuracy": 0.7348242811501597,
       "cc": 57.533374877889166,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 54.639175257731964,
        "score": 52.82931249999998
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 57.533374877889166
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 6.022894779905092,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": -75.0,
        "score": 17.810952380952372
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 6.022894779905092
      },
      "multilingual": {
       "accuracy": 0.6036363636363636,
       "cc": 31.344664586196064,
       "cc_by_type": {
        "choice": 63.808281708509945,
        "noul": -13.888888888888884,
        "score": 44.11460093896714
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 31.344664586196064
      },
      "negation_pair_gen": {
       "accuracy": 0.37209302325581395,
       "cc": -16.322000000000006,
       "cc_by_type": {
        "choice": 36.66666666666667,
        "noul": -66.66666666666667,
        "score": -18.966000000000015
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 69.79409255409256,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 80.0,
        "score": 59.4123076923077
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 69.79409255409256
      },
      "usecase": {
       "accuracy": 0.5363636363636364,
       "cc": 27.856917647058825,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": 0.0,
        "score": 27.68839999999999
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 27.856917647058825
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 32.17107448107448,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": -15.391538461538467
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 32.17107448107448
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 15.06477982590886,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": -0.7349999999999968
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 15.06477982590886
      },
      "de": {
       "accuracy": 0.875,
       "cc": 74.7749634474798,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 73.21869565217392
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 74.7749634474798
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.898075729360645,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.4
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.898075729360645
      },
      "en": {
       "accuracy": 0.577758470894874,
       "cc": 29.89460576976582,
       "cc_by_type": {
        "choice": 55.62660938968279,
        "noul": 4.316546762589923,
        "score": 29.74066115702476
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 29.89460576976582
      },
      "es": {
       "accuracy": 0.6774193548387096,
       "cc": 43.82127882800754,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 50.0,
        "score": -8.748235294117656
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 43.82127882800754
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 50.34222222222223,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 84.36000000000004
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 50.34222222222223
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 11.870773993808049,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": -100.0,
        "score": 77.96526315789474
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 11.870773993808049
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 19.22333333333332,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": -50.0,
        "score": 7.669999999999977
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 19.22333333333332
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 72.32631778198417,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 31.008571428571397
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 72.32631778198417
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 18.961827729238376,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": -19.999999999999996,
        "score": 19.776666666666674
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 18.961827729238376
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -20.761772486772493,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": 26.603571428571414
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.20611111111111,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.61833333333334
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.20611111111111
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -4.4380259740259715,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100.0,
        "score": 64.608
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 32.91830097071453,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -19.999999999999996,
        "score": 41.60720930232558
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 32.91830097071453
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 43.445137339137375,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 19.999999999999996,
        "score": 52.76322580645163
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 43.445137339137375
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 16.34975177304965,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 45.645
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 16.34975177304965
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 29.102952898550726,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 100.0,
        "score": -47.47375
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.102952898550726
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 67.04629032258065,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 65.655
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.04629032258065
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 4.24764595103579,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -28.49999999999997
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 4.24764595103579
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6733333333333333,
       "cc": 57.806310874681884,
       "cc_by_type": {
        "choice": 57.806310874681884
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 57.806310874681884
      },
      "noul": {
       "accuracy": 0.512,
       "cc": 2.400000000000002,
       "cc_by_type": {
        "noul": 2.400000000000002
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 2.400000000000002
      },
      "score": {
       "accuracy": 0.496,
       "cc": 32.36452166064988,
       "cc_by_type": {
        "score": 32.36452166064988
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 32.36452166064988
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 0.5480392156862663,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -50.0,
        "score": -32.3558823529412
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.5480392156862663
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 11.184166180035751,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -11.111111111111116,
        "score": 0.5642307692307891
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 11.184166180035751
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 11.229837398373975,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": -17.530000000000022
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 11.229837398373975
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -20.951396198830405,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": 7.3212500000000125
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 29.18038647342995,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 49.85999999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 29.18038647342995
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -112.2160272536688,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -100.0,
        "score": -273.7550000000001
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 33.790180277239095,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 26.30051282051282
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 33.790180277239095
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": -7.152472352673836,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100.0,
        "score": 30.80739130434782
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 22.21833333333333,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 28.321666666666655
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 22.21833333333333
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 28.726543269958686,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 33.33333333333333,
        "score": 29.934090909090916
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 28.726543269958686
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 38.53218855218855,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 11.111111111111116,
        "score": 4.485454545454548
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 38.53218855218855
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -40.57075268817204,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": 7.319999999999993
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.4444444444444444,
       "cc": 17.760956849363755,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -19.999999999999996,
        "score": 30.864444444444427
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 17.760956849363755
      },
      "model_routing": {
       "accuracy": 0.7603686635944701,
       "cc": 62.05049783549784,
       "cc_by_type": {
        "choice": 70.7792207792208,
        "noul": 72.72727272727273,
        "score": 42.644999999999996
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 62.05049783549784
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 14.542993112792146,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 55.14529411764707
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 14.542993112792146
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 49.24341380377592,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 31.110000000000003
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 49.24341380377592
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 12.83834154351396,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -50.0,
        "score": 32.850000000000016
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 12.83834154351396
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": -1.5517004830917926,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -66.66666666666667,
        "score": 34.475333333333325
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 86.90343947056253,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100.0,
        "score": 82.06263157894738
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 86.90343947056253
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.15564648881318,
      87.27451843980873
     ],
     "I": [
      30.640212312460598,
      38.533368428432574
     ],
     "capability": [
      56.6085688585381,
      62.17252686389763
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 42
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.77132815303852,
     "capability": 61.49805850951083,
     "intelligence": 36.22478886598313
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6733333333333333,
      "calibration": 85.08144262295082,
      "cc": 57.806310874681884,
      "cc_by_type": {
       "choice": 57.806310874681884
      },
      "cc_tiered": 59.522652857590835,
      "cc_tiered_open": 63.253539541495684,
      "cc_tiered_sealed": 55.791766173685986,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.806310874681884
     },
     "noul": {
      "accuracy": 0.512,
      "calibration": 84.0475603217158,
      "cc": 2.400000000000002,
      "cc_by_type": {
       "noul": 2.400000000000002
      },
      "cc_tiered": 12.363699102829539,
      "cc_tiered_open": 24.727398205659078,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 2.400000000000002
     },
     "score": {
      "accuracy": 0.496,
      "calibration": 91.18498151444894,
      "cc": 32.36452166064988,
      "cc_by_type": {
       "score": 32.36452166064988
      },
      "cc_tiered": 36.788014637529024,
      "cc_tiered_open": 46.79054402784911,
      "cc_tiered_sealed": 26.785485247208936,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.36452166064988
     }
    },
    "raw_on_A": {
     "calibration": 85.59486213329072,
     "capability": 60.658649536353764,
     "intelligence": 35.722436939416816
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "guru0381",
   "axes": {
    "calibration": 62.82741496733333,
    "cost": 61.00061799127391,
    "intelligence": 21.99889248592056,
    "speed": 90.96816394625492
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06249555555555558,
      "ece_score": 87.50088888888888,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.50088888888888,
      "score_before_time_limit": 87.50088888888888,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.16533333333333333,
      "brier": 0.2664339086133331,
      "ece": 0.2359746666666666,
      "n": 375,
      "score": 52.805066666666676,
      "score_before_time_limit": 52.805066666666676,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.6186666666666667,
      "interval90_width": 0.2953333333333333,
      "n": 375,
      "nrps": 0.21633341307111098,
      "score": 48.17628934644445,
      "score_before_time_limit": 48.17628934644445,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.4100704
     }
    },
    "score": 62.82741496733333
   },
   "capability": 42.413153726626945,
   "composite_ci95": {
    "A": [
     3.901040583583278,
     14.386971828860638
    ],
    "B": [
     3.0794507040005596,
     12.29364820930768
    ],
    "C": [
     2.7090559608217215,
     9.990952658930997
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: frozen DeepInfra snapshot jevbench-v15-measure-20260925 receipts/DEEPINFRA-MODELS-20260925.json (SHA-256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f); listing Qwen/Qwen3.5-2B, USD 0.02/M input and USD 0.10/M output. The released checkpoint names Qwen/Qwen3.5-2B-Base; the frozen DeepInfra listing omits the -Base suffix. The requester cited the Qwen3.5-4B USD 0.03/M input reference, but the matching 2B family listing is used here (precedent classone-qwen3.5-2b). Self-hosted GPU deployment has no provider tariff, so price_kind is estimate. Kev generates no free-form text. However, kev/api.py output_tokens() tokenizes JSON-serialized answers, so reported usage.output_tokens is nonzero for normal answers; it is not zero by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.019952 per 1,000 decisions (0.309x the official cost cap USD 0.064595), against USD 0.032978 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03297833333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.019951676767676764
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Krino-2B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's `python3 -m krino.serve` (Kev @fe64b127 serve path, documented per-type temperature map choice 1.95 / noul 0.30 / score 0.50 from krino.json), KEV_TRUNCATE_STATES=1, reference kernels (KEV_FUSED=0: the documented pip install has no flash-linear-attention) Paid fast-lane order 8c7dbe11 (public). Kev architecture (LoRA r16 + pointer head) on Qwen/Qwen3.5-2B-Base @b1485b2f. Runtime: reviewed pytorch 2.11 base + Kev uv.lock closure (torch 2.8.0) installed offline from SHA-256-pinned wheels (PR #335). Kev reports usage.output_tokens as the token count of the serialized answer JSON (nothing is generated); priced as reported at the reference output rate.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 26.37590231932532,
    "I_sealed": 17.6218826525158,
    "base": 21.99889248592056,
    "excess": -1.0855071035461918,
    "gap": 8.75401966680952,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 36.29349492821419,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 49.40540540540541,
       "hard": 37.71626297577854,
       "judge": 32.76203032300594,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 21.265010351966872,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 13.043478260869556,
       "judge": -9.999999999999998,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 21.5692016777949,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.23363636363638,
       "hard": 9.25462962962964,
       "judge": 16.76983050847457,
       "standard": 38.56518518518517
      }
     },
     "sealed|choice": {
      "cc": 36.28277020740255,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 46.632782719186785,
       "hard": 30.785243437030175,
       "judge": 35.651322233104814,
       "standard": 43.049989453701755
      }
     },
     "sealed|noul": {
      "cc": 16.58287775014486,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 9.090909090909083,
       "judge": 26.15384615384615,
       "standard": 13.432835820895516
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -5.103771929824585,
       "hard": -28.450816901408405,
       "judge": -18.68588541666667,
       "standard": 12.30559090909088
      }
     }
    }
   },
   "jevbench_score": 8.7238130563062,
   "key": "krino-2b",
   "last_measured_on": "2026-10-12",
   "licence": "Apache-2.0 (adapter, head and code; Kev Apache-2.0; base Qwen3.5-2B-Base Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Guru0381/krino-2b@ff185abf129206174c67ebbe6058cab4cc9ccaed",
   "noul_decisive": {
    "acc_among_decisive": 0.7124600638977636,
    "decisive": 313,
    "decisive_rate": 0.8346666666666667,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 62.82741496733333,
     "cost": 61.00061799127391,
     "intelligence": 21.99889248592056,
     "speed": 91.85485240061512
    },
    "board_release": "v1.6.43",
    "capability": 42.413153726626945,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 109,
    "capability_rank": 137,
    "composite_ci95": {
     "A": [
      3.904931584579009,
      14.406086750913971
     ],
     "B": [
      3.0813923851653424,
      12.304710499013083
     ],
     "C": [
      2.711758044846534,
      10.004226910356923
     ]
    },
    "jevbench_score": 8.734255211215622,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 137,
    "rank": 176,
    "ranks": {
     "A": 176,
     "B": 177,
     "C": 177,
     "capability": 201
    },
    "scores": {
     "A": 8.734255211215622,
     "B": 7.217212347659793,
     "C": 6.065455007788626
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21046640197746455,
     "p50_s_raw": 0.03023320098873228,
     "p95_s_adjusted": 0.30997879329370326,
     "p95_s_raw": 0.07998939664685163
    },
    "views": {
     "Balanced 33:33:33": 7.984144388471859,
     "Emphasis on Accuracy 60:20:20": 5.914448339576854,
     "Emphasis on Cost 20:20:60": 9.172396825433118,
     "Emphasis on Speed 20:60:20": 10.241235033626667,
     "Intelligence only": 4.258556788204026
    }
   },
   "rank": 198,
   "ranked": true,
   "ranks": {
    "A": 198,
    "B": 199,
    "C": 199,
    "capability": 223
   },
   "repo": "https://huggingface.co/Guru0381/krino-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 8.7238130563062,
    "B": 7.211506195412834,
    "C": 6.05820351132375
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21348997401073574,
    "p50_s_raw": 0.031744987005367875,
    "p95_s_adjusted": 0.3748056430893484,
    "p95_s_raw": 0.11240282154467421
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.45454545454545453,
       "cc": -3.9711111111111044,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 50.0,
        "score": -78.57999999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.504950495049505,
       "cc": 31.44464464033227,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 52.0,
        "score": -6.764687500000011
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 31.44464464033227
      },
      "composite_gen": {
       "accuracy": 0.36633663366336633,
       "cc": -31.327821902578023,
       "cc_by_type": {
        "choice": 41.463414634146346,
        "noul": -11.111111111111116,
        "score": -124.3357692307693
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.6041666666666666,
       "cc": 48.12166666666667,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 6.666666666666665,
        "score": 37.69833333333336
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 48.12166666666667
      },
      "jaggedness": {
       "accuracy": 0.35,
       "cc": 1.3896814773217478,
       "cc_by_type": {
        "choice": 17.70933589990377,
        "noul": -18.918918918918916,
        "score": 5.378627450980389
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 1.3896814773217478
      },
      "llm_router": {
       "accuracy": 0.670926517571885,
       "cc": 54.443170038415836,
       "cc_by_type": {
        "choice": 50.86730650700009,
        "noul": 64.94845360824742,
        "score": 47.51374999999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 54.443170038415836
      },
      "long_state": {
       "accuracy": 0.26666666666666666,
       "cc": -24.669193548387096,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100.0,
        "score": 16.315
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -23.08969072164948,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -25.0,
        "score": -45.29999999999998
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5018181818181818,
       "cc": 15.536204017083938,
       "cc_by_type": {
        "choice": 39.3544179980437,
        "noul": 13.888888888888884,
        "score": -6.634694835680777
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 15.536204017083938
      },
      "negation_pair_gen": {
       "accuracy": 0.38372093023255816,
       "cc": -51.32922222222222,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -13.33333333333333,
        "score": -167.321
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.375,
       "cc": 20.106524633821575,
       "cc_by_type": {
        "choice": 20.106524633821575
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 20.106524633821575
      },
      "perturb_pair": {
       "accuracy": 0.625,
       "cc": 45.63698198198199,
       "cc_by_type": {
        "choice": 45.94594594594596,
        "noul": 60.00000000000001,
        "score": 30.964999999999996
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 45.63698198198199
      },
      "usecase": {
       "accuracy": 0.41363636363636364,
       "cc": 5.069023006535965,
       "cc_by_type": {
        "choice": 31.372549019607874,
        "noul": -12.5,
        "score": -3.6654799999999765
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 5.069023006535965
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -33.39369963369963,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50.0,
        "score": -35.895384615384614
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": -9.327886840757811,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": -73.91300000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.625,
       "cc": 45.03688341669872,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 50.0,
        "score": 32.89826086956522
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 45.03688341669872
      },
      "el": {
       "accuracy": 0.26666666666666666,
       "cc": 12.071002482929854,
       "cc_by_type": {
        "choice": -0.5586592178770929,
        "noul": -33.333333333333336,
        "score": 70.10499999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 12.071002482929854
      },
      "en": {
       "accuracy": 0.4778453518679409,
       "cc": 15.730797082087726,
       "cc_by_type": {
        "choice": 35.85628684053157,
        "noul": 17.266187050359715,
        "score": -5.93008264462811
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 15.730797082087726
      },
      "es": {
       "accuracy": 0.5483870967741935,
       "cc": 11.50404119886126,
       "cc_by_type": {
        "choice": 51.06035889070147,
        "noul": 75.0,
        "score": -91.54823529411769
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 11.50404119886126
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 4.108670309653917,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": -22.64666666666666
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 4.108670309653917
      },
      "fr": {
       "accuracy": 0.42857142857142855,
       "cc": 11.742827657378745,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0.0,
        "score": -1.2421052631578888
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 11.742827657378745
      },
      "hi": {
       "accuracy": 0.38095238095238093,
       "cc": -23.99129422718808,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": 0.0,
        "score": -88.175
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 18.527619047619037,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -144.4171428571429
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 18.527619047619037
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 21.515655458476743,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": 39.99999999999999,
        "score": 10.329333333333345
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 21.515655458476743
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -12.233253968253969,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 0.0,
        "score": -3.36642857142857
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.7857142857142857,
       "cc": 79.685,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 72.38833333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 79.685
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 26.801093795093788,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": -0.9820000000000162
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 26.801093795093788
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 42.704345349985395,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 3.3479069767441416
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 42.704345349985395
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 21.2760213786505,
       "cc_by_type": {
        "choice": 4.537418974661164,
        "noul": 60.00000000000001,
        "score": -0.7093548387096638
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 21.2760213786505
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 4.250780141843975,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 34.88
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 4.250780141843975
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 3.729202898550722,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50.0,
        "score": -73.59500000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 3.729202898550722
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.39897849462365,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 64.455
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.39897849462365
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 3.433069679849351,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -5.519999999999969
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 3.433069679849351
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.512,
       "cc": 36.96779502095333,
       "cc_by_type": {
        "choice": 36.96779502095333
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 36.96779502095333
      },
      "noul": {
       "accuracy": 0.5946666666666667,
       "cc": 18.933333333333337,
       "cc_by_type": {
        "noul": 18.933333333333337
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.933333333333337
      },
      "score": {
       "accuracy": 0.328,
       "cc": -6.174584837544961,
       "cc_by_type": {
        "score": -6.174584837544961
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -42.54725490196079,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -50.0,
        "score": -113.64176470588237
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -12.739704039921422,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -33.333333333333336,
        "score": -11.718076923076893
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.12989159891598,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": -2.090000000000014
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.12989159891598
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 34.89788499025341,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 100.0,
        "score": -13.43500000000002
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 34.89788499025341
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 0.4748007246376762,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": -7.271250000000018
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.4748007246376762
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -116.71251572327047,
       "cc_by_type": {
        "choice": -13.207547169811319,
        "noul": -100.0,
        "score": -236.9300000000001
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 11.735613014436538,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -42.85714285714286,
        "score": 27.08358974358973
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 11.735613014436538
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -12.115642074433163,
       "cc_by_type": {
        "choice": -14.98257839721254,
        "noul": -50.0,
        "score": 28.63565217391304
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 26.11814814814815,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 60.00000000000001,
        "score": 1.687777777777777
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 26.11814814814815
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -3.2785867237687394,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0.0,
        "score": -19.900000000000006
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 31.765082217524082,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": -14.782272727272705
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 31.765082217524082
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 9.02948476702508,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": -41.728750000000026
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 9.02948476702508
      },
      "llm_guardrails": {
       "accuracy": 0.2962962962962963,
       "cc": -19.13207400298573,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -60.0,
        "score": -28.29833333333338
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.6728110599078341,
       "cc": 52.803651357323226,
       "cc_by_type": {
        "choice": 59.0909090909091,
        "noul": 60.606060606060595,
        "score": 38.713984374999995
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 52.803651357323226
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 16.1079067404313,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 0.0,
        "score": 5.905294117647076
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 16.1079067404313
      },
      "recruiting": {
       "accuracy": 0.3333333333333333,
       "cc": -14.35318239173391,
       "cc_by_type": {
        "choice": 33.147632311977716,
        "noul": -33.333333333333336,
        "score": -42.87384615384611
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.2222222222222222,
       "cc": -30.597434560030916,
       "cc_by_type": {
        "choice": 11.330049261083742,
        "noul": -50.0,
        "score": -53.12235294117649
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": 11.78285507246376,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": 0.0,
        "score": 7.81233333333331
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 11.78285507246376
      },
      "search_retrieval": {
       "accuracy": 0.5416666666666666,
       "cc": 29.112966223387648,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 0.0,
        "score": 19.367368421052632
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 29.112966223387648
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      56.134235634833225,
      65.29815028441786
     ],
     "I": [
      16.102596267666183,
      26.8828021176775
     ],
     "capability": [
      36.956661624305724,
      45.50971580905799
     ],
     "composite": {
      "A": [
       3.901040583583278,
       14.386971828860638
      ],
      "B": [
       3.0794507040005596,
       12.29364820930768
      ],
      "C": [
       2.7090559608217215,
       9.990952658930997
      ]
     },
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.82741496733333,
     "capability": 42.413153726626945,
     "intelligence": 21.99889248592056
    },
    "per_type": {
     "choice": {
      "accuracy": 0.512,
      "calibration": 87.50088888888888,
      "cc": 36.96779502095333,
      "cc_by_type": {
       "choice": 36.96779502095333
      },
      "cc_tiered": 36.28813256780837,
      "cc_tiered_open": 36.29349492821419,
      "cc_tiered_sealed": 36.28277020740255,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 36.96779502095333
     },
     "noul": {
      "accuracy": 0.5946666666666667,
      "calibration": 52.805066666666676,
      "cc": 18.933333333333337,
      "cc_by_type": {
       "noul": 18.933333333333337
      },
      "cc_tiered": 18.923944051055866,
      "cc_tiered_open": 21.265010351966872,
      "cc_tiered_sealed": 16.58287775014486,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.933333333333337
     },
     "score": {
      "accuracy": 0.328,
      "calibration": 48.17628934644445,
      "cc": -6.174584837544961,
      "cc_by_type": {
       "score": -6.174584837544961
      },
      "cc_tiered": 10.78460083889745,
      "cc_tiered_open": 21.5692016777949,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 63.43795542912849,
     "capability": 42.556073742724294,
     "intelligence": 21.674192056320095
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 7.972513302254613,
    "Emphasis on Accuracy 60:20:20": 5.910615727547576,
    "Emphasis on Cost 20:20:60": 9.16318221352688,
    "Emphasis on Speed 20:60:20": 10.206854437734147,
    "Intelligence only": 4.258556788204026
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Launchables",
   "axes": {
    "calibration": 84.04872871623093,
    "cost": 57.35738348807602,
    "intelligence": 57.25409127121432,
    "speed": 85.292646608578
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.058081079642959894,
      "ece_score": 88.38378407140802,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.38378407140802,
      "score_before_time_limit": 88.38378407140802,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.15128528819337783,
      "ece": 0.10546966253208946,
      "n": 375,
      "score": 78.90606749358211,
      "score_before_time_limit": 78.90606749358211,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.944,
      "interval90_width": 0.3946666666666667,
      "n": 375,
      "nrps": 0.08944470329719607,
      "score": 84.85633458370268,
      "score_before_time_limit": 84.85633458370268,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10671430251437525
     }
    },
    "score": 84.04872871623093
   },
   "capability": 70.65140999372262,
   "composite_ci95": {
    "A": [
     65.64091834901892,
     69.8069877215792
    ],
    "B": [
     62.331726606578606,
     67.89525971382662
    ],
    "C": [
     48.87458483878408,
     69.8069877215792
    ]
   },
   "cost": {
    "basis": "ESTIMATE, nearest-listed-sibling reference: declared base google/gemma-4-12B-it is UNLISTED in frozen pricing_v15 (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7); no exact base reference exists. Same-base published precedents arcbright-elf-1-12b, chon-1.0-12b and metask_jev_rain_12b_trained use google/gemma-3-12b-it, USD 0.05/M input and 0.15/M output. This row uses that same sibling ESTIMATE, not a vendor bill or quote. Launchables list price GBP 0.0015 per million input tokens, output free, is shown for information and is not used for the Cost axis. Applied to provider-reported usage. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 779,294 input / 1,485 output tokens. All-item usage 989,607 input / 1,500 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.033136900000000004
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02638885521885522
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Myles-G 12B (Launchables)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane; full S1200+P300 draw v1.6-regular-20261012, unchanged evaluator API runner from Sandy, customer API key; full-set exposure gate logged all sealed items before the first scored request. Native probabilities; latency has no API adjustment.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 60.47280030970968,
    "I_sealed": 54.035382232718966,
    "base": 57.25409127121432,
    "excess": -3.402108693364994,
    "gap": 6.437418076990717,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 71.8314160213165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 71.6892104435357,
       "judge": 56.49307844429796,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 55.719806763285035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 53.867178144527514,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.40372091886416,
       "hard": 53.51144632562129,
       "judge": 50.74381636500394,
       "standard": 54.99541306445699
      }
     },
     "sealed|choice": {
      "cc": 65.72076424974752,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 63.05076905285068,
       "judge": 60.33300685602351,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 56.64273760207883,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 49.09090909090909,
       "judge": 50.76923076923077,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 39.74264484633055,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 56.81348601357418,
       "hard": 39.23170405307833,
       "judge": 29.28973440649999,
       "standard": 47.908471508959
      }
     }
    }
   },
   "jevbench_score": 68.34949057703662,
   "key": "launchables-myles-g-12b",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary vendor-hosted API; weights and serving code not independently inspected",
   "listing": "ranked",
   "measured_in": "v1.6.38",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://www.launchables.ai/licensing#launchables-myles-g-12b",
   "noul_decisive": {
    "acc_among_decisive": 0.7946666666666666,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 18,
    "axes": {
     "calibration": 84.04872871623093,
     "cost": 57.35738348807602,
     "intelligence": 57.25409127121432,
     "speed": 85.64932567577961
    },
    "board_release": "v1.6.43",
    "capability": 70.65140999372262,
    "capability_api_board_rank": 18,
    "capability_open_board_rank": null,
    "capability_rank": 43,
    "composite_ci95": {
     "A": [
      65.69355393933306,
      69.86651967043076
     ],
     "B": [
      62.36968910887525,
      67.94030392837915
     ],
     "C": [
      48.91386791946196,
      69.86651967043076
     ]
    },
    "jevbench_score": 68.40656153142275,
    "measured_in": "v1.6.38",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 34,
    "ranks": {
     "A": 34,
     "B": 39,
     "C": 41,
     "capability": 74
    },
    "scores": {
     "A": 68.40656153142275,
     "B": 65.84152005408374,
     "C": 62.28856307389066
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.479910550522618,
     "p50_s_raw": 0.479910550522618,
     "p95_s_adjusted": 0.5674232601770199,
     "p95_s_raw": 0.5674232601770199
    },
    "views": {
     "Balanced 33:33:33": 64.4107683660651,
     "Emphasis on Accuracy 60:20:20": 61.34362333393462,
     "Emphasis on Cost 20:20:60": 61.39100451654052,
     "Emphasis on Speed 20:60:20": 71.50304365746292,
     "Intelligence only": 57.25409127121432
    }
   },
   "rank": 40,
   "ranked": true,
   "ranks": {
    "A": 40,
    "B": 43,
    "C": 44,
    "capability": 87
   },
   "repo": "https://www.launchables.ai/licensing",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.34949057703662,
    "B": 65.79921496246162,
    "C": 62.23659630838769
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4840246535022743,
    "p50_s_raw": 0.4840246535022743,
    "p95_s_adjusted": 0.6107561607146635,
    "p95_s_raw": 0.6107561607146635
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API through the unchanged TypeSafe adapter; no vendor code ran locally. TRUNCATION ON THE 15 LONGEST ITEMS (scored as answered): the provider silently truncated their input and reported 12,511 to 15,221 tokens; it returned answers, not refusals, under its 16,384-token context. The scored-wrong rule applies only to refusals, so these items are scored as answered, as for earlier provider-truncated rows. SAME-AUTHOR SECOND use after Myles-G 31B full-set exposure; the public set is treated as seen. Customer states merged LoRA on google/gemma-4-12B-it; deployed weights, serving and training claims are unverified. Measured accuracy on the 15 shortened states: 2 of 15 correct (13.3 %), versus 1083 of 1485 (72.9 %) on the remaining items; a Score item is correct within half a label step.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 73.02371545408378,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 44.071146362251355
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 73.02371545408378
      },
      "calibration_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 55.34888443858599,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": 84.00000000000001,
        "score": 39.31070421713652
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 55.34888443858599
      },
      "composite_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 41.59179534449972,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 3.703703703703698,
        "score": 47.0879424923971
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 41.59179534449972
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 85.71737204456754,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 86.66666666666667,
        "score": 70.48544946703595
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.71737204456754
      },
      "jaggedness": {
       "accuracy": 0.5666666666666667,
       "cc": 34.81576106951519,
       "cc_by_type": {
        "choice": 48.0269489894129,
        "noul": 40.54054054054055,
        "score": 15.879793678592124
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 34.81576106951519
      },
      "llm_router": {
       "accuracy": 0.8466453674121406,
       "cc": 75.82535881523177,
       "cc_by_type": {
        "choice": 76.22611605177423,
        "noul": 91.75257731958763,
        "score": 59.497383074333456
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 75.82535881523177
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -46.41598515353363,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100.0,
        "score": -36.022149008987995
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 41.50206138678643,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 50.0,
        "score": 23.990720242833497
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 41.50206138678643
      },
      "multilingual": {
       "accuracy": 0.7345454545454545,
       "cc": 60.13778089654668,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 55.55555555555556,
        "score": 56.15873268348128
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 60.13778089654668
      },
      "negation_pair_gen": {
       "accuracy": 0.6627906976744186,
       "cc": 29.911992335055242,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": 0.0,
        "score": 13.069310338499063
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 29.911992335055242
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 74.00836593423553,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100.0,
        "score": 52.0551278327366
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.00836593423553
      },
      "usecase": {
       "accuracy": 0.6545454545454545,
       "cc": 52.849677358848375,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": 54.16666666666667,
        "score": 41.147071292231374
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.849677358848375
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 3.819060228511955,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0.0,
        "score": -12.352343123987941
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 3.819060228511955
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 52.10545103048704,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 43.72034694706789
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.10545103048704
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 85.79316708994303,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 81.27330657956358
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.79316708994303
      },
      "el": {
       "accuracy": 0.6,
       "cc": 43.00860821915449,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 95.8787110634225
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 43.00860821915449
      },
      "en": {
       "accuracy": 0.7185056472632494,
       "cc": 55.84520341950083,
       "cc_by_type": {
        "choice": 68.36748392135803,
        "noul": 57.55395683453237,
        "score": 41.614169502612086
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 55.84520341950083
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.18505736183396,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100.0,
        "score": 27.91895675108097
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.18505736183396
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 53.040130879712194,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 91.36082979760654
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 53.040130879712194
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 75.9551412851916,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 70.2183650320454
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.9551412851916
      },
      "hi": {
       "accuracy": 1.0,
       "cc": 81.96851164064229,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0,
        "score": 45.90553492192687
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 81.96851164064229
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 58.47867787426869,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 3.4952697880437866
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 58.47867787426869
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 47.9045972019261,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 60.00000000000001,
        "score": 40.90203624438037
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 47.9045972019261
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 48.894562335279964,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 46.6836870058399
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 48.894562335279964
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.62497577118171,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 93.54159398021181
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.62497577118171
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 36.93305405397785,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 29.413880776652157
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 36.93305405397785
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 59.45596416808655,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 53.60276343104764
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 59.45596416808655
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 73.31967475908746,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 62.38683806630188
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.31967475908746
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 27.84815255959744,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": 60.00000000000001,
        "score": 25.672117253260396
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 27.84815255959744
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 65.84902838886349,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 19.28621560137309
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 65.84902838886349
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 67.08568154736894,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 65.77317367436487
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.08568154736894
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 66.36968910397002,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -0.8909326880899515
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 66.36968910397002
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.752,
       "cc": 67.96724009261563,
       "cc_by_type": {
        "choice": 67.96724009261563
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 67.96724009261563
      },
      "noul": {
       "accuracy": 0.7946666666666666,
       "cc": 58.93333333333332,
       "cc_by_type": {
        "noul": 58.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 58.93333333333332
      },
      "score": {
       "accuracy": 0.5946666666666667,
       "cc": 44.11986046405637,
       "cc_by_type": {
        "score": 44.11986046405637
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 44.11986046405637
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 21.436138948141217,
       "cc_by_type": {
        "choice": 52.0,
        "noul": 0.0,
        "score": 12.308416844423641
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 21.436138948141217
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 59.65610142196325,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 77.77777777777777,
        "score": 19.824066860782764
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 59.65610142196325
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 44.27306472524267,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 15.339519378980015
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.27306472524267
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 18.13548546873989,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -8.751438330622419
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 18.13548546873989
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 22.251914893223415,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 29.07458525938037
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.251914893223415
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 15.587403181990245,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -69.5899917118909
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 15.587403181990245
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 38.185896214716806,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 39.48766063294595
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 38.185896214716806
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 1.5120896906177588,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100.0,
        "score": 56.801077434222606
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 1.5120896906177588
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 52.701004610275824,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100.0,
        "score": 20.603013830827464
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 52.701004610275824
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 59.81133967013818,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 100.0,
        "score": 30.825881965446655
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 59.81133967013818
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 48.49298180723741,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 1.5512968429008622
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 48.49298180723741
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 63.20796045838467,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 21.88194589128305
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 63.20796045838467
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 83.43877597847415,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 100.0,
        "score": 61.83264271469309
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 83.43877597847415
      },
      "model_routing": {
       "accuracy": 0.8571428571428571,
       "cc": 76.57582004023605,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 100.0,
        "score": 49.59758999083801
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 76.57582004023605
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 42.92899893712856,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0.0,
        "score": 63.335941149197595
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 42.92899893712856
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.976710098847924,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 65.02298080722161
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.976710098847924
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 51.03200757320826,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50.0,
        "score": 32.65267296593021
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 51.03200757320826
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 23.916868816794192,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": 33.33333333333333,
        "score": 39.86654847936809
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 23.916868816794192
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 91.10806316661156,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100.0,
        "score": 94.67650266709447
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 91.10806316661156
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.91555558047827,
      85.87719178871096
     ],
     "I": [
      51.80648252445323,
      61.39756475428746
     ],
     "capability": [
      65.8773400775301,
      72.96432355292742
     ],
     "composite": {
      "A": [
       65.64091834901892,
       69.8069877215792
      ],
      "B": [
       62.331726606578606,
       67.89525971382662
      ],
      "C": [
       48.87458483878408,
       69.8069877215792
      ]
     },
     "seed": 46
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.04872871623093,
     "capability": 70.65140999372262,
     "intelligence": 57.25409127121432
    },
    "per_type": {
     "choice": {
      "accuracy": 0.752,
      "calibration": 88.38378407140802,
      "cc": 67.96724009261563,
      "cc_by_type": {
       "choice": 67.96724009261563
      },
      "cc_tiered": 68.776090135532,
      "cc_tiered_open": 71.8314160213165,
      "cc_tiered_sealed": 65.72076424974752,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.96724009261563
     },
     "noul": {
      "accuracy": 0.7946666666666666,
      "calibration": 78.90606749358211,
      "cc": 58.93333333333332,
      "cc_by_type": {
       "noul": 58.93333333333332
      },
      "cc_tiered": 56.18127218268194,
      "cc_tiered_open": 55.719806763285035,
      "cc_tiered_sealed": 56.64273760207883,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 58.93333333333332
     },
     "score": {
      "accuracy": 0.5946666666666667,
      "calibration": 84.85633458370268,
      "cc": 44.11986046405637,
      "cc_by_type": {
       "score": 44.11986046405637
      },
      "cc_tiered": 46.80491149542903,
      "cc_tiered_open": 53.867178144527514,
      "cc_tiered_sealed": 39.74264484633055,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 44.11986046405637
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.3433182404572,
    "Emphasis on Accuracy 60:20:20": 61.306899259848294,
    "Emphasis on Cost 20:20:60": 61.354223706959694,
    "Emphasis on Speed 20:60:20": 71.35358067835261,
    "Intelligence only": 57.25409127121432
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Launchables",
   "axes": {
    "calibration": 83.99531596546265,
    "cost": 51.11161516579885,
    "intelligence": 73.64186833978687,
    "speed": 85.94853930142175
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06430449201022093,
      "ece_score": 87.13910159795581,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.13910159795581,
      "score_before_time_limit": 87.13910159795581,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.08560855771666369,
      "ece": 0.06318301734560669,
      "n": 375,
      "score": 87.36339653087866,
      "score_before_time_limit": 87.36339653087866,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.824,
      "interval90_width": 0.114,
      "n": 375,
      "nrps": 0.07920606342697788,
      "score": 77.48344976755348,
      "score_before_time_limit": 77.48344976755348,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.18556247061097633
     }
    },
    "score": 83.99531596546265
   },
   "capability": 78.81859215262476,
   "composite_ci95": {
    "A": [
     68.95161343706428,
     71.55692630209987
    ],
    "B": [
     68.92716284510078,
     72.62891518757193
    ],
    "C": [
     68.95161343706428,
     71.55692630209987
    ]
   },
   "cost": {
    "basis": "ESTIMATE, v1.5-M2 base-model reference rule. Launchables publishes a list price for Myles-G 31B (launchables.ai/licensing, fetched 21 Oct 2026, saved copy sha256 ad4736d5...): GBP 0.005 per million input tokens, output free; that is USD 0.0066734 per million input tokens at the ECB reference rate of 20 Oct 2026 (GBP 1 = USD 1.33468), prices exclude VAT. Under the method, the base-model reference price applies as an ESTIMATE for this vendor-hosted row, as for attest-31b-v1.0: base model google/gemma-4-31B-it through the frozen pricing_v15 module (sha256 1f4d1f7b1ce97ab27b938043444b9c2757e4a9dbecc799a803122588401f5bf1) reference() -> google/gemma-4-31b-it on OpenRouter, USD 0.09 per million input tokens, USD 0.34 per million output tokens. The vendor list price is shown for information and is not used for the Cost axis. Applied to this row's own provider-reported tokens (907,107 input and 1,500 output tokens over all 1,500 items; 697,619 input and 1,485 output tokens over the 1,485 items of the common cost basis; the 15 truncated items count for no row's cost). Estimated, not a vendor bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them in full).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05476641999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0426199393939394
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Myles-G 31B (Launchables)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane order 46550a0f. Vendor-hosted Launchables API (api.launchables.ai, model `myles-g-31b`, TypeSafe-compatible endpoint), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) over the official per-item latency subset without API adjustment; one non-item warm-up request preceded the scored loop and is excluded. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `launchables` (no proxy) before the first request. All 1,500 of 1,500 requests answered with HTTP 200 and the served model was `myles-g-31b` on every row; native probabilities on all three question types. Run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 76.07886474383956,
    "I_sealed": 71.20487193573419,
    "base": 73.64186833978687,
    "excess": -4.9655339622503405,
    "gap": 4.8739928081053705,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.98663238745361,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 79.2387543252595,
       "judge": 72.31377719182598,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 87.52173913043478,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 70.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 60.72822271363029,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 80.5682624678971,
       "hard": 65.56227672741873,
       "judge": 55.992001550320516,
       "standard": 48.24442655388464
      }
     },
     "sealed|choice": {
      "cc": 80.48409588326635,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 78.66312015727996,
       "judge": 77.96278158667972,
       "standard": 82.91499683611053
      }
     },
     "sealed|noul": {
      "cc": 75.46738696198295,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 67.27272727272727,
       "judge": 81.53846153846153,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 57.66313296195325,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 77.78722486429184,
       "hard": 61.05229981148842,
       "judge": 48.908706111046754,
       "standard": 53.95439358807338
      }
     }
    }
   },
   "jevbench_score": 70.56596417573436,
   "key": "launchables-myles-g-31b",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary hosted API (no public weights release; the merged adapter is gated on Hugging Face); declared base google/gemma-4-31B-it is under the Gemma 4 licence",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.launchables.ai/m/myles-g-31b#myles-g-31b",
   "noul_decisive": {
    "acc_among_decisive": 0.8933333333333333,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 15,
    "axes": {
     "calibration": 83.99531596546265,
     "cost": 51.11161516579885,
     "intelligence": 73.64186833978687,
     "speed": 86.60775159362743
    },
    "board_release": "v1.6.43",
    "capability": 78.81859215262476,
    "capability_api_board_rank": 8,
    "capability_open_board_rank": null,
    "capability_rank": 16,
    "composite_ci95": {
     "A": [
      69.05703328640422,
      71.67046972209141
     ],
     "B": [
      69.01141312494278,
      72.72246397695918
     ],
     "C": [
      69.05703328640422,
      71.67046972209141
     ]
    },
    "jevbench_score": 70.6763821133842,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 24,
    "ranks": {
     "A": 24,
     "B": 14,
     "C": 22,
     "capability": 31
    },
    "scores": {
     "A": 70.6763821133842,
     "B": 71.25021746899355,
     "C": 70.6763821133842
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4372560594929382,
     "p50_s_raw": 0.4372560594929382,
     "p95_s_adjusted": 0.49944653444108555,
     "p95_s_raw": 0.49944653444108555
    },
    "views": {
     "Balanced 33:33:33": 67.12826022407477,
     "Emphasis on Accuracy 60:20:20": 69.59036149121279,
     "Emphasis on Cost 20:20:60": 59.65119627360718,
     "Emphasis on Speed 20:60:20": 73.7646036645773,
     "Intelligence only": 73.64186833978687
    }
   },
   "rank": 24,
   "ranked": true,
   "ranks": {
    "A": 24,
    "B": 16,
    "C": 22,
    "capability": 36
   },
   "repo": "https://www.launchables.ai/licensing",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.56596417573436,
    "B": 71.16041573983628,
    "C": 70.56596417573436
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4449399680015631,
    "p50_s_raw": 0.4449399680015631,
    "p95_s_adjusted": 0.5712742523639462,
    "p95_s_raw": 0.5712742523639462
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no vendor code ran locally. Serving and training code are proprietary (customer statement); the endpoint is TypeSafe-compatible and was driven by the unchanged TypeSafe adapter. Model, customer statement: the merged LoRA adapter launchables/myles-g-31b@0093c518 (gated on Hugging Face) on google/gemma-4-31B-it@842da379. Neither the deployed weights nor the serving stack were independently inspected. TRUNCATION ON THE 15 LONGEST ITEMS (scored as answered): the 15 longest items of this draw (about 77,000 to 82,000 tokens on other tokenizers) were answered, not refused. The endpoint reported only 12,456 to 15,166 input tokens for them, against at most 1,774 for every other item, so it silently truncated their input to its documented 16,384-token context; it returned no refusal and no truncation flag. They are scored as the answers the endpoint gave, as for krino-2b, metask_jev_rain_31b_trained and reflex-instinct-0.6b, which also answered these items on shortened input; the scored-wrong rule applies only to refusals. The cost of the truncation shows up in accuracy, measured: on those 15 items 4 of 15 answers are correct (26.7 %) against 1,234 of 1,485 (83.1 %) on the other 1,485 items (Choice 3 of 10 vs 637 of 740; Noul 0 of 2 vs 335 of 373; Score 1 of 3 vs 262 of 372; a Score item counts as correct within half a label step). MODEL, customer statement: the merged LoRA adapter launchables/myles-g-31b@0093c518 (gated on Hugging Face) on google/gemma-4-31B-it@842da379; serving and training code are proprietary and the API is TypeSafe-compatible. TRAINING AND SELECTION, customer statement: the 231 public items entered model selection (not training); the public set is therefore treated as seen during selection and the normal public-vs-sealed gap rule applies to this row. Customer statements about the model are not independently verified.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 88.81777977591078,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 91.45333932773237
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 88.81777977591078
      },
      "calibration_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 69.49843983156292,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 52.0,
        "score": 72.40200030062722
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 69.49843983156292
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 84.38274189852417,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 85.18518518518519,
        "score": 74.4671055510377
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 84.38274189852417
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 99.98441015141567,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.95323045424703
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.98441015141567
      },
      "jaggedness": {
       "accuracy": 0.6722222222222223,
       "cc": 54.01241281618022,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 67.56756756756756,
        "score": 32.005763277508215
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 54.01241281618022
      },
      "llm_router": {
       "accuracy": 0.939297124600639,
       "cc": 85.62782046706222,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 95.8762886597938,
        "score": 66.55441232931219
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 85.62782046706222
      },
      "long_state": {
       "accuracy": 0.26666666666666666,
       "cc": -41.32275847934246,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100.0,
        "score": -33.645694792866095
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 65.95778331068276,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 100.0,
        "score": 47.357886014522464
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 65.95778331068276
      },
      "multilingual": {
       "accuracy": 0.76,
       "cc": 66.0209651405287,
       "cc_by_type": {
        "choice": 70.6553635474405,
        "noul": 69.44444444444444,
        "score": 57.963087429701154
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 66.0209651405287
      },
      "negation_pair_gen": {
       "accuracy": 0.8488372093023255,
       "cc": 69.28604230653808,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 54.52479358628089
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 69.28604230653808
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 85.15390034237629,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100.0,
        "score": 73.47971904514688
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.15390034237629
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 70.61828397537191,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 79.16666666666667,
        "score": 54.74700878886081
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 70.61828397537191
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 37.55745153351111,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100.0,
        "score": -30.184788256609508
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 37.55745153351111
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 75.70280420891712,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 47.84573981569144
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 75.70280420891712
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 88.6005763604217,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 81.73093262108807
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.6005763604217
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.60445955038642,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.99959839045162
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.60445955038642
      },
      "en": {
       "accuracy": 0.8392701998262381,
       "cc": 74.81183747161053,
       "cc_by_type": {
        "choice": 83.52473120904064,
        "noul": 79.13669064748201,
        "score": 61.77409055830891
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.81183747161053
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 70.84890494528481,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 32.122571279573876
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 70.84890494528481
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.8458841272402,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 99.32453762762225
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.8458841272402
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 80.55022576235007,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 84.00361846352078
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 80.55022576235007
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 75.82065942079434,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100.0,
        "score": 44.221754798695876
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 75.82065942079434
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 58.74455539879628,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -9.736715720992283
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 58.74455539879628
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 40.29914618031202,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": -10.508439139762915
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 40.29914618031202
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 80.34273879226296,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0,
        "score": 85.47266082123335
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 80.34273879226296
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.77706128881198,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.99785053310259
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.77706128881198
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 39.98237054931828,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 38.56183026267348
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.98237054931828
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 67.66762593881772,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 60.00000000000001,
        "score": 58.237748743241156
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.66762593881772
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.6267366064763,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 43.7010701612085
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.6267366064763
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 50.842845370067124,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 43.59236589743543
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 50.842845370067124
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 80.7096626100372,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 63.86811826489422
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 80.7096626100372
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 70.22321689159186,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 42.927715190904635
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 70.22321689159186
      },
      "zh": {
       "accuracy": 1.0,
       "cc": 85.7067013690126,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 57.120104107037825
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 85.7067013690126
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8533333333333334,
       "cc": 81.0558946784286,
       "cc_by_type": {
        "choice": 81.0558946784286
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.0558946784286
      },
      "noul": {
       "accuracy": 0.8933333333333333,
       "cc": 78.66666666666666,
       "cc_by_type": {
        "noul": 78.66666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 78.66666666666666
      },
      "score": {
       "accuracy": 0.7013333333333334,
       "cc": 59.69977832737404,
       "cc_by_type": {
        "score": 59.69977832737404
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 59.69977832737404
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7727272727272727,
       "cc": 66.27948690725826,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 75.0,
        "score": 39.83846072177478
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 66.27948690725826
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 67.70707644780173,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 21.754769716076005
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 67.70707644780173
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 30.911133285722144,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": -58.48611233795551
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.911133285722144
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 23.698433433647455,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 100.0,
        "score": -47.03335466981786
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 23.698433433647455
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 54.16742916528917,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 72.64721503209941
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 54.16742916528917
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": 33.620491366367595,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 66.66666666666667,
        "score": -28.06934351096011
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.620491366367595
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 62.696753361107994,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 71.42857142857142,
        "score": 36.26953179200747
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 62.696753361107994
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.401951415293944,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50.0,
        "score": 68.56473926330344
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.401951415293944
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 45.037407616672056,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100.0,
        "score": -2.3877771499838385
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 45.037407616672056
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 77.07005858523151,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100.0,
        "score": 56.90610723321059
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 77.07005858523151
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 66.1587119553411,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 32.32626506498971
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.1587119553411
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 76.64975776840441,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 94.46540233747132
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 76.64975776840441
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 77.70507938454888,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 73.11523815364664
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 77.70507938454888
      },
      "model_routing": {
       "accuracy": 0.9308755760368663,
       "cc": 83.66920067248388,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100.0,
        "score": 56.85175786160749
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 83.66920067248388
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 46.62074774073085,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 62.89487278073378
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 46.62074774073085
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 60.69597414311201,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 82.18077294001385
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 60.69597414311201
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 38.31899566170807,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50.0,
        "score": -5.486362768570374
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 38.31899566170807
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 48.074426494813274,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 68.8609606438601
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 48.074426494813274
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.99391370850913,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.98174112552738
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.99391370850913
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.117788381695,
      86.00359922013621
     ],
     "I": [
      68.58224521990246,
      77.30034626422231
     ],
     "capability": [
      75.03476101966254,
      81.19655150762269
     ],
     "composite": {
      "A": [
       68.95161343706428,
       71.55692630209987
      ],
      "B": [
       68.92716284510078,
       72.62891518757193
      ],
      "C": [
       68.95161343706428,
       71.55692630209987
      ]
     },
     "seed": 44
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.99531596546265,
     "capability": 78.81859215262476,
     "intelligence": 73.64186833978687
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8533333333333334,
      "calibration": 87.13910159795581,
      "cc": 81.0558946784286,
      "cc_by_type": {
       "choice": 81.0558946784286
      },
      "cc_tiered": 80.23536413535999,
      "cc_tiered_open": 79.98663238745361,
      "cc_tiered_sealed": 80.48409588326635,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.0558946784286
     },
     "noul": {
      "accuracy": 0.8933333333333333,
      "calibration": 87.36339653087866,
      "cc": 78.66666666666666,
      "cc_by_type": {
       "noul": 78.66666666666666
      },
      "cc_tiered": 81.49456304620887,
      "cc_tiered_open": 87.52173913043478,
      "cc_tiered_sealed": 75.46738696198295,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 78.66666666666666
     },
     "score": {
      "accuracy": 0.7013333333333334,
      "calibration": 77.48344976755348,
      "cc": 59.69977832737404,
      "cc_by_type": {
       "score": 59.69977832737404
      },
      "cc_tiered": 59.19567783779177,
      "cc_tiered_open": 60.72822271363029,
      "cc_tiered_sealed": 57.66313296195325,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 59.69977832737404
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.99550249391719,
    "Emphasis on Accuracy 60:20:20": 69.50469257996821,
    "Emphasis on Cost 20:20:60": 59.58823989428031,
    "Emphasis on Speed 20:60:20": 73.47661299008504,
    "Intelligence only": 73.64186833978687
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 91.25367387086692,
    "cost": 62.300793859384434,
    "intelligence": 69.70041168706942,
    "speed": 88.94430045323229
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.038374286188809156,
      "ece_score": 92.32514276223817,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 92.32514276223817,
      "score_before_time_limit": 92.32514276223817,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.14477211796246647,
      "brier": 0.0787106673820862,
      "ece": 0.060856253290503357,
      "n": 373,
      "score": 87.82874934189933,
      "score_before_time_limit": 87.82874934189933,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.26276881720430106,
      "n": 372,
      "nrps": 0.05673524666969873,
      "score": 93.60712950846329,
      "score_before_time_limit": 93.60712950846329,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03556108158051766
     }
    },
    "score": 91.25367387086692
   },
   "capability": 80.47704277896817,
   "composite_ci95": {
    "A": [
     73.89351069961306,
     76.97573052365466
    ],
    "B": [
     71.94286014570793,
     76.23436422046282
    ],
    "C": [
     73.89351069961306,
     76.97573052365466
    ]
   },
   "cost": {
    "basis": "Liquid AI public price list, USD 0.042/M input, output free (fetched 16 Oct 2026): the models table of Liquid's public pricing page lists d1-ultra at USD 0.042 per 1M input tokens and USD 0.00 per 1M output tokens (saved copy sha256 89fb8ef2...); the same rate Liquid's API billed on the measurement's smoke call (48 input tokens -> USD 0.000002016). Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops and SubQ Snap), through a scoped re-freeze of the frozen pricing module that adds this one key and nothing else, with a zero-difference proof for every other registry entry. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (638,436 input tokens over the 1,485 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis the scorer uses; the 15 items refused with HTTP 422 count as zero tokens and as wrong answers. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.018057 per 1,000 decisions (0.280x the official cost cap USD 0.064595), against USD 0.017876 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.017876208
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01805677575757576
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "d1-ultra (Liquid AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.liquid.ai/decisions/v1/systemone API, model `d1-ultra`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `liquid` before the first request (Florian GO 15 Oct 2026). Native probabilities on all three question types. 15 of 1,500 items exceed the model's 32,768-token input limit (HTTP 422) and are scored as failures.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 75.72153851823367,
    "I_sealed": 63.679284855905166,
    "base": 69.70041168706942,
    "excess": 2.2027268919727945,
    "gap": 12.042253662328505,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 88.89079890008239,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 88.13447593935399,
       "standard": 100.0
      }
     },
     "open|noul": {
      "cc": 75.17046238785369,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 63.10335426676494,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.60054737282905,
       "hard": 68.18905752206545,
       "judge": 75.94709495533684,
       "standard": 35.917740170274
      }
     },
     "sealed|choice": {
      "cc": 79.10284436684508,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 72.93859141898925,
       "judge": 79.72575905974536,
       "standard": 84.81333052098715
      }
     },
     "sealed|noul": {
      "cc": 53.0885768888857,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 43.636363636363626,
       "judge": 53.846153846153854,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 58.84643331198474,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 81.7986845210918,
       "hard": 56.171536782426415,
       "judge": 58.372812971884905,
       "standard": 53.43053127669758
      }
     }
    }
   },
   "jevbench_score": 76.04601536374408,
   "key": "liquid-d1-ultra",
   "last_measured_on": "2026-10-15",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.21",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.liquid.ai/decisions/v1/systemone#d1-ultra",
   "noul_decisive": {
    "acc_among_decisive": 0.9373040752351097,
    "decisive": 319,
    "decisive_rate": 0.8552278820375335,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 3,
    "axes": {
     "calibration": 91.25367387086692,
     "cost": 62.300793859384434,
     "intelligence": 69.70041168706942,
     "speed": 88.99977436791615
    },
    "board_release": "v1.6.43",
    "capability": 80.47704277896817,
    "capability_api_board_rank": 6,
    "capability_open_board_rank": null,
    "capability_rank": 11,
    "composite_ci95": {
     "A": [
      73.90307802669203,
      76.98611269050402
     ],
     "B": [
      71.95011503809144,
      76.24251050830729
     ],
     "C": [
      73.90307802669203,
      76.98611269050402
     ]
    },
    "jevbench_score": 76.05614823638027,
    "measured_in": "v1.6.21",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 4,
    "ranks": {
     "A": 4,
     "B": 5,
     "C": 4,
     "capability": 22
    },
    "scores": {
     "A": 76.05614823638027,
     "B": 74.69393258837212,
     "C": 76.05614823638027
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.2313630650169216,
     "p50_s_raw": 0.2313630650169216,
     "p95_s_adjusted": 0.5441624051978579,
     "p95_s_raw": 0.5441624051978579
    },
    "views": {
     "Balanced 33:33:33": 72.05604070021411,
     "Emphasis on Accuracy 60:20:20": 71.0949370716979,
     "Emphasis on Cost 20:20:60": 67.80894855156113,
     "Emphasis on Speed 20:60:20": 77.99554201006464,
     "Intelligence only": 69.70041168706942
    }
   },
   "rank": 5,
   "ranked": true,
   "ranks": {
    "A": 5,
    "B": 7,
    "C": 5,
    "capability": 28
   },
   "repo": "https://www.liquid.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 76.04601536374408,
    "B": 74.68611384483172,
    "C": 76.04601536374408
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2306590935040731,
    "p50_s_raw": 0.2306590935040731,
    "p95_s_adjusted": 0.5528398923954227,
    "p95_s_raw": 0.5528398923954227
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 80.02551491048317,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 48.40987806478281
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 80.02551491048317
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 53.5127160507667,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": 60.00000000000001,
        "score": 45.07685440892786
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 53.5127160507667
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 89.03033227085523,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 81.00246926062347
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.03033227085523
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.90669247576356,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 97.72007742729068
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.90669247576356
      },
      "jaggedness": {
       "accuracy": 0.7111111111111111,
       "cc": 52.548678213136746,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 35.13513513513513,
        "score": 60.04699190081021
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 52.548678213136746
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 80.3197048932887,
       "cc_by_type": {
        "choice": 91.28290921898389,
        "noul": 89.69072164948453,
        "score": 59.98548381139766
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.3197048932887
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 40.01561610820875,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 0.0,
        "score": 44.789116365863336
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 40.01561610820875
      },
      "multilingual": {
       "accuracy": 0.7418181818181818,
       "cc": 55.53031034406329,
       "cc_by_type": {
        "choice": 80.43690903162701,
        "noul": 25.0,
        "score": 61.15402200056285
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 55.53031034406329
      },
      "negation_pair_gen": {
       "accuracy": 0.9302325581395349,
       "cc": 76.76565063912945,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 80.0,
        "score": 50.296951917388334
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.76565063912945
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 80.13869805419485,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 52.42810617459654
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.13869805419485
      },
      "usecase": {
       "accuracy": 0.7681818181818182,
       "cc": 61.12254914211481,
       "cc_by_type": {
        "choice": 80.39215686274511,
        "noul": 41.66666666666667,
        "score": 61.30882389693265
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 61.12254914211481
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 27.790054081448684,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 0.0,
        "score": 2.417781291965093
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.790054081448684
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 70.84904875775025,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 100.0,
        "score": 54.021800651130945
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 70.84904875775025
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 89.08391810246775,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 83.18095784722625
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 89.08391810246775
      },
      "el": {
       "accuracy": 0.6,
       "cc": 44.35440994419011,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 99.91611623852936
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.35440994419011
      },
      "en": {
       "accuracy": 0.8236316246741964,
       "cc": 69.98283524145941,
       "cc_by_type": {
        "choice": 80.88868820248715,
        "noul": 67.62589928057554,
        "score": 61.433918241315546
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 69.98283524145941
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.08416705224533,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100.0,
        "score": 27.616285822315124
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.08416705224533
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 76.59925311455045,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 96.46442601031806
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 76.59925311455045
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 96.12745113668352,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 88.38235341005056
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 96.12745113668352
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 69.2291292495233,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 50.0,
        "score": 57.68738774856992
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 69.2291292495233
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 62.7322379583571,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -11.803286124928691
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 62.7322379583571
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 24.203419893141085,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 0.0,
        "score": -13.092679160926245
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 24.203419893141085
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 18.422112493954483,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 55.26633748186346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.422112493954483
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.4730866883171,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.41926006495127
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.4730866883171
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -5.121222995410733,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 10.610356987793757
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 66.50566804059815,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 67.13443958518846
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 66.50566804059815
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 85.83933080677154,
       "cc_by_type": {
        "choice": 89.39304655274012,
        "noul": 100.0,
        "score": 68.1249458675745
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 85.83933080677154
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 45.882356110583025,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 43.17898322536608
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 45.882356110583025
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 47.56476228162157,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0.0,
        "score": 64.43341727964732
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 47.56476228162157
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.73277592080343,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 68.45639227853934
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.73277592080343
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 46.87044056187991,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 7.277988352306409
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.87044056187991
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8546666666666667,
       "cc": 81.22811381771561,
       "cc_by_type": {
        "choice": 81.22811381771561
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.22811381771561
      },
      "noul": {
       "accuracy": 0.7973333333333333,
       "cc": 59.46666666666667,
       "cc_by_type": {
        "noul": 59.46666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 59.46666666666667
      },
      "score": {
       "accuracy": 0.7333333333333333,
       "cc": 60.44867908600179,
       "cc_by_type": {
        "score": 60.44867908600179
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 60.44867908600179
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 33.23171209119251,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -25.0,
        "score": 40.695136273577546
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 33.23171209119251
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 94.09748592551544,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 82.29245777654631
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 94.09748592551544
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 62.92582030524064,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": 37.557948720599974
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.92582030524064
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": -17.965299307538217,
       "cc_by_type": {
        "choice": 64.91228070175438,
        "noul": -100.0,
        "score": -18.80817862436903
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 24.49212514702329,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 64.78072326715683
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 24.49212514702329
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": -13.88384228808446,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -124.67039478878168
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 35.72504937468765,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -42.85714285714286,
        "score": 59.83621254983327
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 35.72504937468765
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 31.77084188905293,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50.0,
        "score": 76.6714106845804
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 31.77084188905293
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 45.1815821381559,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 19.999999999999996,
        "score": 57.211413081134374
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 45.1815821381559
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 76.48497795483895,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 62.7882671978502
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 76.48497795483895
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 66.48644556865014,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 33.309465904916834
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.48644556865014
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -11.843346976830077,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": 26.835550467359237
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 55.32046425195082,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 45.96139275585243
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.32046425195082
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 79.30093337735978,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 93.93939393939394,
        "score": 47.46989969917894
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.30093337735978
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 20.596744504671886,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 84.82286307255691
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 20.596744504671886
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.36102452879326,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 63.17592409705759
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.36102452879326
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 17.81076588752592,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50.0,
        "score": 18.21062278573048
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 17.81076588752592
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 24.11187241023282,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": -66.66666666666667,
        "score": 53.495037520553545
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 24.11187241023282
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.89828165670299,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.69484497010896
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.89828165670299
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.25252044978473,
      91.63978255866996
     ],
     "I": [
      64.56901745362967,
      73.4726823695734
     ],
     "capability": [
      76.18500946956924,
      81.95767090067203
     ],
     "composite": {
      "A": [
       73.89351069961306,
       76.97573052365466
      ],
      "B": [
       71.94286014570793,
       76.23436422046282
      ],
      "C": [
       73.89351069961306,
       76.97573052365466
      ]
     },
     "seed": 45
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.25367387086692,
     "capability": 80.47704277896817,
     "intelligence": 69.70041168706942
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 92.32514276223817,
      "cc": 81.22811381771561,
      "cc_by_type": {
       "choice": 81.22811381771561
      },
      "cc_tiered": 83.99682163346372,
      "cc_tiered_open": 88.89079890008239,
      "cc_tiered_sealed": 79.10284436684508,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.22811381771561
     },
     "noul": {
      "accuracy": 0.7973333333333333,
      "calibration": 87.82874934189933,
      "cc": 59.46666666666667,
      "cc_by_type": {
       "noul": 59.46666666666667
      },
      "cc_tiered": 64.12951963836969,
      "cc_tiered_open": 75.17046238785369,
      "cc_tiered_sealed": 53.0885768888857,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 59.46666666666667
     },
     "score": {
      "accuracy": 0.7333333333333333,
      "calibration": 93.60712950846329,
      "cc": 60.44867908600179,
      "cc_by_type": {
       "score": 60.44867908600179
      },
      "cc_tiered": 60.97489378937484,
      "cc_tiered_open": 63.10335426676494,
      "cc_tiered_sealed": 58.84643331198474,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.44867908600179
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.0439144024908,
    "Emphasis on Accuracy 60:20:20": 71.08785360500364,
    "Emphasis on Cost 20:20:60": 67.80250471482547,
    "Emphasis on Speed 20:60:20": 77.96997204149442,
    "Intelligence only": 69.70041168706942
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Messier AI (@agentmessier-ai)",
   "axes": {
    "calibration": 72.86180997007926,
    "cost": 61.121663950171616,
    "intelligence": 52.02542301918905,
    "speed": 89.42792459288492
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12514095842876055,
      "ece_score": 74.9718083142479,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 74.9718083142479,
      "score_before_time_limit": 74.9718083142479,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.17675601582379594,
      "ece": 0.15402190415708572,
      "n": 375,
      "score": 69.19561916858285,
      "score_before_time_limit": 69.19561916858285,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.856,
      "interval90_width": 0.33844444444444444,
      "n": 375,
      "nrps": 0.11129842749409077,
      "score": 74.41800242740706,
      "score_before_time_limit": 74.41800242740706,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.2001707619788841
     }
    },
    "score": 72.86180997007926
   },
   "capability": 62.443616494634156,
   "composite_ci95": {
    "A": [
     54.20534555244628,
     68.23272412517842
    ],
    "B": [
     50.58133631549522,
     65.48115342913778
    ],
    "C": [
     37.64260107808769,
     60.86497996482708
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 dcfcecb0..., the module in force; its base-model references are unchanged since the dae86740... module this row was first priced with - the two later re-freezes each added one hosted-API launch-list key and nothing else): declared base Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output, the same basis the live messier-one-v0.2 row carries. The author's server reports NO usage block of any kind, so the input tokens are counted OFFLINE with the author's own Prompter (prompt.py at tag v0.4) at the exact serving settings (--max-state 7500 --max-total 8150, AA labels, qboth layout) and cross-checked against vLLM's own prompt-token counters from the measured pass; output tokens are 0 (the engine is asked for one token restricted to the option labels and nothing is generated for the caller). The reference listing was deprecated after the 25 Sep 2026 cut-off and replaced by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3). Estimated, not charged, not a quote from the author or any host. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.019767 per 1,000 decisions (0.306x the official cost cap USD 0.064595), against USD 0.022118 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.022117700000000004
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01976717171717172
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Messier One v0.4 (Qwen3.5-4B fine-tune, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX A6000 48 GB - different hardware from the H100/RTX 5090 rows of this draw and from the RTX 3090 the author quotes. Started by the author's OWN ./serve_vllm.sh MODEL_DIR 8080 with its defaults, which is what the author insists on for v0.4: vLLM 0.30.0, --max-model-len 8192, chunked prefill and prefix caching OFF (this revision is non-causal), --max-logprobs 256, processed_logprobs, front end --max-state 7500 --max-total 8150 --gate 0.5 --noul-floor 0.801; transformers 5.18.0 exactly as pinned. One request at a time from the unchanged runner. Isolation: the server ran inside a network namespace with loopback only (unshare -n), the cut proven before the sealed input was staged and re-proven immediately before the first sealed request, with the synthetic smoke gate passed in between; a network namespace replaces docker --network none because the pod carried no local image (custody #19335, same property, proven the same way). The pod host kept its network for ssh.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 55.631843562186205,
    "I_sealed": 48.419002476191885,
    "base": 52.02542301918905,
    "excess": -2.626685684361391,
    "gap": 7.21284108599432,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.70061629878168,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 75.46398238439761,
       "judge": 64.40342781806197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 59.21394064872326,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 30.000000000000004,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 34.98097373905368,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 43.1256839171765,
       "hard": 43.01475509215395,
       "judge": 34.883418864196,
       "standard": 14.987388256078226
      }
     },
     "sealed|choice": {
      "cc": 60.22807849887394,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 56.80582861107898,
       "judge": 54.162585700293846,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 51.03745559638509,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 68.96551724137932,
       "hard": 50.90909090909091,
       "judge": 38.46153846153846,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 33.991473333316634,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 53.18191088973121,
       "hard": 30.414088756811818,
       "judge": 29.47815246036215,
       "standard": 38.32100501755069
      }
     }
    }
   },
   "jevbench_score": 66.12774376240803,
   "key": "messier-one-v0.4",
   "last_measured_on": "2026-10-14",
   "licence": "Weights CC BY-NC 4.0 (non-commercial: free for research and evaluation, commercial licence on request; listing a measured score is evaluation use); code Apache-2.0; base Qwen/Qwen3.5-4B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "agentmessier/messier-one@14c84de57b71bf48608e2d886d86ebc5c6f8897e",
   "noul_decisive": {
    "acc_among_decisive": 0.7786666666666666,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 72.86180997007926,
     "cost": 61.121663950171616,
     "intelligence": 52.02542301918905,
     "speed": 89.67127141858641
    },
    "board_release": "v1.6.43",
    "capability": 62.443616494634156,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 49,
    "capability_rank": 74,
    "composite_ci95": {
     "A": [
      54.231114077287856,
      68.26806273860754
     ],
     "B": [
      50.59939794983643,
      65.50718704517482
     ],
     "C": [
      37.66049588700546,
      60.89651165228654
     ]
    },
    "jevbench_score": 66.16093508471093,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": 24,
    "rank": 45,
    "ranks": {
     "A": 45,
     "B": 48,
     "C": 51,
     "capability": 116
    },
    "scores": {
     "A": 66.16093508471093,
     "B": 62.75099656762108,
     "C": 49.74281676276681
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2828623611479998,
     "p50_s_raw": 0.06643118057399988,
     "p95_s_adjusted": 0.3813271221704781,
     "p95_s_raw": 0.11566356108523905
    },
    "views": {
     "Balanced 33:33:33": 64.19305626178894,
     "Emphasis on Accuracy 60:20:20": 58.70145093265181,
     "Emphasis on Cost 20:20:60": 62.92818786712826,
     "Emphasis on Speed 20:60:20": 72.42418258466023,
     "Intelligence only": 52.02542301918905
    }
   },
   "rank": 51,
   "ranked": true,
   "ranks": {
    "A": 51,
    "B": 54,
    "C": 58,
    "capability": 133
   },
   "repo": "https://huggingface.co/agentmessier/messier-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.12774376240803,
    "B": 62.72710717607955,
    "C": 49.71786200870668
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2880231073126197,
    "p50_s_raw": 0.06901155365630984,
    "p95_s_adjusted": 0.396077533904463,
    "p95_s_raw": 0.1230387669522315
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 82.16341854240056,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 71.49025562720168
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 82.16341854240056
      },
      "calibration_gen": {
       "accuracy": 0.5148514851485149,
       "cc": 29.04494301665566,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": 43.99999999999999,
        "score": 9.942888434908658
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 29.04494301665566
      },
      "composite_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 46.36869268140659,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 55.55555555555556,
        "score": 9.566782651265848
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 46.36869268140659
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.14562852488994,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 89.4368855746698
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.14562852488994
      },
      "jaggedness": {
       "accuracy": 0.5611111111111111,
       "cc": 33.94067092331888,
       "cc_by_type": {
        "choice": 49.4706448508181,
        "noul": 24.32432432432432,
        "score": 28.027043594814227
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 33.94067092331888
      },
      "llm_router": {
       "accuracy": 0.7795527156549521,
       "cc": 66.83382437622242,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 75.25773195876289,
        "score": 56.14979030259788
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 66.83382437622242
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": 5.043826168411282,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0.0,
        "score": 5.454059150395141
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 5.043826168411282
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": 10.394403866583607,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": 25.0,
        "score": -7.218850255919285
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 10.394403866583607
      },
      "multilingual": {
       "accuracy": 0.6909090909090909,
       "cc": 53.25250361364891,
       "cc_by_type": {
        "choice": 66.7427453537659,
        "noul": 55.55555555555556,
        "score": 37.459209931625296
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 53.25250361364891
      },
      "negation_pair_gen": {
       "accuracy": 0.6395348837209303,
       "cc": 24.730034373509014,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 26.66666666666666,
        "score": -19.143230212806284
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 24.730034373509014
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 80.45660602129784,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100.0,
        "score": 62.39083908491454
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.45660602129784
      },
      "usecase": {
       "accuracy": 0.65,
       "cc": 52.043880329569305,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": 58.33333333333333,
        "score": 35.788503733805946
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.043880329569305
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 17.790541563258085,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 100.0,
        "score": -51.39028007213051
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 17.790541563258085
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 59.56093069774181,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 45.34945875989212
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 59.56093069774181
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 72.21420782877351,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75.0,
        "score": 57.57182702614354
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.21420782877351
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 54.75552840589854,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 97.78613829032132
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 54.75552840589854
      },
      "en": {
       "accuracy": 0.6828844483058211,
       "cc": 51.49058721733788,
       "cc_by_type": {
        "choice": 63.754408659889414,
        "noul": 54.67625899280575,
        "score": 36.04109399931846
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 51.49058721733788
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 68.24190669801142,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 4.725720094034247
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.24190669801142
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 33.934589289450244,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": 100.0,
        "score": 32.95130885195729
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 33.934589289450244
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 33.62990228437019,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0.0,
        "score": 64.41911861781645
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 33.62990228437019
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 32.11300719789304,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25.0,
        "score": 4.858574666304816
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 32.11300719789304
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 64.97069379895709,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 22.971317562109007
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 64.97069379895709
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 39.099214004618965,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 60.00000000000001,
        "score": 14.485886652458968
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 39.099214004618965
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 32.83013966262374,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": -1.509581012128769
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 32.83013966262374
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.18530071199275,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.5559021359783
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.18530071199275
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 44.550106052514856,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 52.265036772263194
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 44.550106052514856
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -14.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 69.82411392080512,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 44.70721268920334
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 69.82411392080512
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 62.10287688730801,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 39.34339789822337
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.10287688730801
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 26.944485862981143,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 37.429202269794494
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 26.944485862981143
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 69.43378739054388,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 30.040492606414237
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 69.43378739054388
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 55.79040730990699,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -32.62877807027902
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 55.79040730990699
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 35.76198975772707,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -0.6236352465927908
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 35.76198975772707
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7213333333333334,
       "cc": 64.00619988901434,
       "cc_by_type": {
        "choice": 64.00619988901434
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 64.00619988901434
      },
      "noul": {
       "accuracy": 0.7786666666666666,
       "cc": 55.73333333333332,
       "cc_by_type": {
        "noul": 55.73333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 55.73333333333332
      },
      "score": {
       "accuracy": 0.52,
       "cc": 35.63329245619988,
       "cc_by_type": {
        "score": 35.63329245619988
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.63329245619988
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 31.602907151164914,
       "cc_by_type": {
        "choice": 36.0,
        "noul": 50.0,
        "score": 8.808721453494739
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 31.602907151164914
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 24.65440579911538,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 55.55555555555556,
        "score": -25.69171704019699
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 24.65440579911538
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 58.85440387413508,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 100.0,
        "score": 57.86402463053535
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 58.85440387413508
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 21.573024553922433,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": 24.952991790422274
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 21.573024553922433
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 28.246437211964235,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": 33.33333333333333,
        "score": 23.869746418501414
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 28.246437211964235
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": 0.38882349872808203,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": -69.2737810761428
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.38882349872808203
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 42.964161901689614,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 71.42857142857142,
        "score": -3.320399448992739
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 42.964161901689614
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 14.800234981182662,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50.0,
        "score": 46.665513305917315
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 14.800234981182662
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 34.22439466186014,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 64.33985065224708
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.22439466186014
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 35.392748862215676,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 3.7514086137705016
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 35.392748862215676
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 49.47199754097719,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": 38.338473243086646
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 49.47199754097719
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 30.394104568777696,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 22.365109405257822
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 30.394104568777696
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 56.36973577985058,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 60.62552211882236
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 56.36973577985058
      },
      "model_routing": {
       "accuracy": 0.7972350230414746,
       "cc": 66.80548991484399,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 78.78787878787878,
        "score": 42.66755199561424
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.80548991484399
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 48.12446743220059,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0.0,
        "score": 55.889717075872404
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 48.12446743220059
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 80.53804875918388,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": 58.32723819955723
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 80.53804875918388
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 40.26737295094906,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50.0,
        "score": 0.35876909915260047
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 40.26737295094906
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 41.32801314674797,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 19.636213353287392
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 41.32801314674797
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 79.25826168668284,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100.0,
        "score": 59.12709822730834
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 79.25826168668284
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.2065286202611,
      76.13123675368387
     ],
     "I": [
      46.451591543620204,
      56.656336267839215
     ],
     "capability": [
      57.76581304676275,
      65.61559579468398
     ],
     "composite": {
      "A": [
       54.20534555244628,
       68.23272412517842
      ],
      "B": [
       50.58133631549522,
       65.48115342913778
      ],
      "C": [
       37.64260107808769,
       60.86497996482708
      ]
     },
     "seed": 46
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 72.86180997007926,
     "capability": 62.443616494634156,
     "intelligence": 52.02542301918905
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7213333333333334,
      "calibration": 74.9718083142479,
      "cc": 64.00619988901434,
      "cc_by_type": {
       "choice": 64.00619988901434
      },
      "cc_tiered": 66.46434739882781,
      "cc_tiered_open": 72.70061629878168,
      "cc_tiered_sealed": 60.22807849887394,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 64.00619988901434
     },
     "noul": {
      "accuracy": 0.7786666666666666,
      "calibration": 69.19561916858285,
      "cc": 55.73333333333332,
      "cc_by_type": {
       "noul": 55.73333333333332
      },
      "cc_tiered": 55.12569812255417,
      "cc_tiered_open": 59.21394064872326,
      "cc_tiered_sealed": 51.03745559638509,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 55.73333333333332
     },
     "score": {
      "accuracy": 0.52,
      "calibration": 74.41800242740706,
      "cc": 35.63329245619988,
      "cc_by_type": {
       "score": 35.63329245619988
      },
      "cc_tiered": 34.48622353618516,
      "cc_tiered_open": 34.98097373905368,
      "cc_tiered_sealed": 33.991473333316634,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.63329245619988
     }
    },
    "raw_on_A": {
     "calibration": 76.02710478723333,
     "capability": 64.44192558101349,
     "intelligence": 52.85674637479366
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.15140078129758,
    "Emphasis on Accuracy 60:20:20": 58.68054487347202,
    "Emphasis on Cost 20:20:60": 62.90416339698564,
    "Emphasis on Speed 20:60:20": 72.32880521310358,
    "Intelligence only": 52.02542301918905
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "MeTask (Hangzhou MeTask Technology Co., Ltd.)",
   "axes": {
    "calibration": 87.29383335481249,
    "cost": 52.66805268932843,
    "intelligence": 66.48910229704197,
    "speed": 71.7899932169731
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04579106842955546,
      "ece_score": 90.8417863140889,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 89.87280726007195,
      "score_before_time_limit": 90.8417863140889,
      "time_limit_failed_share": 0.010666666666666666
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.12338284879474153,
      "ece": 0.05903423423423422,
      "n": 375,
      "score": 88.19315315315316,
      "score_before_time_limit": 88.19315315315316,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9226666666666666,
      "interval90_width": 0.28755555555555545,
      "n": 375,
      "nrps": 0.07999688319975698,
      "score": 83.81553965121238,
      "score_before_time_limit": 84.71921123774837,
      "time_limit_failed_share": 0.010666666666666666,
      "top_ece": 0.11280944602263793
     }
    },
    "score": 87.29383335481249
   },
   "capability": 76.89146782592724,
   "composite_ci95": {
    "A": [
     65.50048371626353,
     68.30801088615483
    ],
    "B": [
     64.63214467446335,
     68.68812503612823
    ],
    "C": [
     65.50048371626353,
     68.30801088615483
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: base google/gemma-4-12B-it (author int4 AWQ of google/gemma-4-12B-it-qat-q4_0-unquantized + own LoRA). google/gemma-4-12b-it is not listed on OpenRouter; nearest listed sibling of the same family and size class (fixed Gemma-4-12B precedent) OpenRouter google/gemma-3-12b-it USD 0.05/M input, USD 0.15/M output, read 2026-10-12T00:13:38Z from https://openrouter.ai/api/v1/models x the service's own measured usage tokens; estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.037821 per 1,000 decisions (0.586x the official cost cap USD 0.064595), against USD 0.041404 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.041404166666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03782087542087542
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 12,
    "n_over_limit_answered_ok": 12,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-12B-trained",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline read-only container (--network none); author's own service (serve.sh -> launch.py -> server.py on vLLM 0.31.0, pinned digest image), unchanged defaults Paid fast-lane order 66c7b6d2 (public). Own LoRA adapter (rank 16) on the author's int4 Gemma 4 12B base, applied unmerged by vLLM. Replaces nothing: the earlier metask_jev_rain_12b row (third-party AWQ wrapper, order f1b81eec) is a different system.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 71.19688579914401,
    "I_sealed": 61.781318794939914,
    "base": 66.48910229704197,
    "excess": -0.42395976615161146,
    "gap": 9.4155670042041,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.70868696671687,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 77.35136835482857,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 79.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 70.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 54.28293661429005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.59298352366106,
       "hard": 53.42979629970057,
       "judge": 51.83502724515816,
       "standard": 48.50605784248132
      }
     },
     "sealed|choice": {
      "cc": 68.44910042331313,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 64.09159245981265,
       "judge": 62.97747306562194,
       "standard": 77.2199957814807
      }
     },
     "sealed|noul": {
      "cc": 66.55391237686658,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 52.727272727272734,
       "judge": 69.23076923076923,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 50.340943584640044,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 62.57017720623247,
       "hard": 48.706025420365464,
       "judge": 47.129277578702755,
       "standard": 52.31366211129891
      }
     }
    }
   },
   "jevbench_score": 67.32648682650284,
   "key": "metask_jev_rain_12b_trained",
   "last_measured_on": "2026-10-12",
   "licence": "Weights: Apache-2.0 per model card (base Gemma 4 under the Gemma 4 licence); service code MIT (github.com/chenyuwang166-oss/metask-jev-rain-12B-trained@e36c7a54)",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "MeTask-Rain/metask-jev-rain-12B-trained@cf3706fdd57e23025a8c97fbde86bd5b4a547010",
   "noul_decisive": {
    "acc_among_decisive": 0.856,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.91805023499681,
     "cost": 52.66805268932843,
     "intelligence": 67.8255189476327,
     "speed": 73.37579625429389
    },
    "board_release": "v1.6.43",
    "capability": 77.87178459131476,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 9,
    "capability_rank": 18,
    "composite_ci95": {
     "A": [
      66.36559898881377,
      68.99113382462001
     ],
     "B": [
      65.70425913953832,
      69.45086045975164
     ],
     "C": [
      66.36559898881377,
      68.99113382462001
     ]
    },
    "jevbench_score": 68.10451633937213,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 18,
    "rank": 37,
    "ranks": {
     "A": 37,
     "B": 29,
     "C": 31,
     "capability": 34
    },
    "scores": {
     "A": 68.10451633937213,
     "B": 68.0485333889376,
     "C": 68.10451633937213
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3790490360464901,
     "p50_s_raw": 0.11452451802324504,
     "p95_s_adjusted": 12.126207013113888,
     "p95_s_raw": 5.988103506556944
    },
    "views": {
     "Balanced 33:33:33": 63.34589532500726,
     "Emphasis on Accuracy 60:20:20": 65.06481097061045,
     "Emphasis on Cost 20:20:60": 58.59417683415789,
     "Emphasis on Speed 20:60:20": 67.00978182322413,
     "Intelligence only": 67.8255189476327
    }
   },
   "rank": 47,
   "ranked": true,
   "ranks": {
    "A": 47,
    "B": 36,
    "C": 38,
    "capability": 48
   },
   "repo": "https://huggingface.co/MeTask-Rain/metask-jev-rain-12B-trained",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.32648682650284,
    "B": 67.15732675508717,
    "C": 67.32648682650284
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.40733872305136176,
    "p50_s_raw": 0.12866936152568087,
    "p95_s_adjusted": 16.257171256013677,
    "p95_s_raw": 8.05358562800684
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1.0,
       "cc": 87.84887353749237,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 63.546620612477135
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.84887353749237
      },
      "calibration_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 56.940570475966105,
       "cc_by_type": {
        "choice": 45.91728525980912,
        "noul": 76.0,
        "score": 48.9044261680892
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 56.940570475966105
      },
      "composite_gen": {
       "accuracy": 0.7821782178217822,
       "cc": 64.2194805783471,
       "cc_by_type": {
        "choice": 64.22764227642277,
        "noul": 55.55555555555556,
        "score": 72.875243903063
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 64.2194805783471
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 89.4731058334587,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 68.41931750037614
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 89.4731058334587
      },
      "jaggedness": {
       "accuracy": 0.6555555555555556,
       "cc": 51.15173057621822,
       "cc_by_type": {
        "choice": 53.801732435033685,
        "noul": 56.75675675675676,
        "score": 42.8967025368642
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 51.15173057621822
      },
      "llm_router": {
       "accuracy": 0.8370607028753994,
       "cc": 71.72047864231678,
       "cc_by_type": {
        "choice": 78.6035044465968,
        "noul": 87.62886597938144,
        "score": 48.92906550097207
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.72047864231678
      },
      "long_state": {
       "accuracy": 0.26666666666666666,
       "cc": -31.382028062735532,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100.0,
        "score": -3.8235035430453035
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 31.665606480992682,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": 50.0,
        "score": 6.85248954607085
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 31.665606480992682
      },
      "multilingual": {
       "accuracy": 0.8,
       "cc": 69.17212800571714,
       "cc_by_type": {
        "choice": 75.54613628953375,
        "noul": 77.77777777777777,
        "score": 54.192469949839904
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.17212800571714
      },
      "negation_pair_gen": {
       "accuracy": 0.7790697674418605,
       "cc": 55.42410495913857,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 39.99999999999999,
        "score": 42.9389815440824
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 55.42410495913857
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 80.61760708445547,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100.0,
        "score": 62.873842274387414
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.61760708445547
      },
      "usecase": {
       "accuracy": 0.7272727272727273,
       "cc": 59.31205332037768,
       "cc_by_type": {
        "choice": 66.91176470588238,
        "noul": 58.33333333333333,
        "score": 52.691061921917324
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.31205332037768
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 24.545761058228774,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -19.21985968245654
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.545761058228774
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 79.27519011200793,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 58.56289752496387
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 79.27519011200793
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 85.48969822591152,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 72.39829821755762
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.48969822591152
      },
      "el": {
       "accuracy": 0.8,
       "cc": 76.94633377862073,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 97.6918877418212
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 76.94633377862073
      },
      "en": {
       "accuracy": 0.7697654213727194,
       "cc": 64.4827040827019,
       "cc_by_type": {
        "choice": 71.00352692791152,
        "noul": 68.34532374100719,
        "score": 54.099261579186994
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 64.4827040827019
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.5487504069601,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 30.434179442740017
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.5487504069601
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 65.70550033996419,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 100.0,
        "score": 95.47715675759747
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.70550033996419
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 83.74288210729817,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 72.40511691012979
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 83.74288210729817
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 59.09382926533001,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100.0,
        "score": -5.958735667697135
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 59.09382926533001
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 48.56239452957701,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -40.28319832865008
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 48.56239452957701
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 45.094346455841844,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 39.99999999999999,
        "score": 52.471284006127604
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.094346455841844
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 34.600352011262636,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 48.24550047823236
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 34.600352011262636
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 98.74153284520177,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.22459853560528
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.74153284520177
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 48.51424476202488,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 12.209400952741323
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.51424476202488
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 71.6454228057721,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 50.17113934410426
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 71.6454228057721
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 81.35337503204035,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100.0,
        "score": 65.27403199064081
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 81.35337503204035
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 37.10841544635961,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": 60.00000000000001,
        "score": 78.98482080716391
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 37.10841544635961
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 37.768161021349435,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -86.6955169359517
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 37.768161021349435
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 65.4423185501839,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -3.6730443494483067
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 65.4423185501839
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 37.32160001214934,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -21.368533296885307
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 37.32160001214934
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.78,
       "cc": 71.5838420176429,
       "cc_by_type": {
        "choice": 71.5838420176429
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.5838420176429
      },
      "noul": {
       "accuracy": 0.856,
       "cc": 71.2,
       "cc_by_type": {
        "noul": 71.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 71.2
      },
      "score": {
       "accuracy": 0.672,
       "cc": 52.02832383682014,
       "cc_by_type": {
        "score": 52.02832383682014
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 52.02832383682014
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 39.183979710642106,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 25.0,
        "score": 24.551939131926314
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 39.183979710642106
      },
      "code_linting": {
       "accuracy": 0.7083333333333334,
       "cc": 51.97732888297762,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 77.77777777777777,
        "score": -3.2122507561741065
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 51.97732888297762
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 42.34789039430175,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50.0,
        "score": -6.696166215468713
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 42.34789039430175
      },
      "demand_forecasting": {
       "accuracy": 0.5882352941176471,
       "cc": 40.03575445852952,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": 45.253462205997906
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 40.03575445852952
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 30.104630020806553,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 67.12548426531822
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 30.104630020806553
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -40.6229056009301,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -204.8875847273186
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 54.499913068666864,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 40.25043948611262
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 54.499913068666864
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 44.23391278756434,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 0.0,
        "score": 43.154700035166904
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 44.23391278756434
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 65.41586316119422,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 100.0,
        "score": 79.580922816916
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 65.41586316119422
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 44.35835074057644,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 30.648214248852803
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 44.35835074057644
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 72.23152413124878,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 100.0,
        "score": 28.322479370490505
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 72.23152413124878
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -1.6851463390176065,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": -73.87264331812808
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 72.19021405060475,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 68.08695693108486
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.19021405060475
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 74.29999007860982,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 96.96969696969697,
        "score": 35.28092261678184
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.29999007860982
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 69.91229706647958,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 100.0,
        "score": 44.28583553725066
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 69.91229706647958
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 76.15530393930493,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 100.0,
        "score": 78.6051875839315
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 76.15530393930493
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 68.0715834514785,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 100.0,
        "score": 48.54972572389362
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 68.0715834514785
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 30.843083819184248,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": 33.33333333333333,
        "score": 60.645193486538254
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 30.843083819184248
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.96186264566033,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 95.56174452061086
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.96186264566033
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.1317029634105,
      88.88837341971802
     ],
     "I": [
      61.61085998902087,
      70.65774547957805
     ],
     "capability": [
      72.50621643606394,
      79.04031154182296
     ],
     "composite": {
      "A": [
       65.50048371626353,
       68.30801088615483
      ],
      "B": [
       64.63214467446335,
       68.68812503612823
      ],
      "C": [
       65.50048371626353,
       68.30801088615483
      ]
     },
     "seed": 47
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.29383335481249,
     "capability": 76.89146782592724,
     "intelligence": 66.48910229704197
    },
    "per_type": {
     "choice": {
      "accuracy": 0.78,
      "calibration": 89.87280726007195,
      "cc": 71.5838420176429,
      "cc_by_type": {
       "choice": 71.5838420176429
      },
      "cc_tiered": 74.078893695015,
      "cc_tiered_open": 79.70868696671687,
      "cc_tiered_sealed": 68.44910042331313,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.5838420176429
     },
     "noul": {
      "accuracy": 0.856,
      "calibration": 88.19315315315316,
      "cc": 71.2,
      "cc_by_type": {
       "noul": 71.2
      },
      "cc_tiered": 73.07647309664586,
      "cc_tiered_open": 79.59903381642512,
      "cc_tiered_sealed": 66.55391237686658,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 71.2
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 83.81553965121238,
      "cc": 52.02832383682014,
      "cc_by_type": {
       "score": 52.02832383682014
      },
      "cc_tiered": 52.31194009946505,
      "cc_tiered_open": 54.28293661429005,
      "cc_tiered_sealed": 50.340943584640044,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.02832383682014
     }
    },
    "raw_on_A": {
     "calibration": 85.53121656093457,
     "capability": 75.98626480220324,
     "intelligence": 66.44131304347191
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.010666666666666713,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 62.55679800481707,
    "Emphasis on Accuracy 60:20:20": 64.07255178517673,
    "Emphasis on Cost 20:20:60": 58.18682661084714,
    "Emphasis on Speed 20:60:20": 65.94961439371524,
    "Intelligence only": 66.48910229704197
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "MeTask (Hangzhou MeTask Technology Co., Ltd.)",
   "axes": {
    "calibration": 87.09471194592824,
    "cost": 50.73365709012802,
    "intelligence": 70.71905450578407,
    "speed": 82.33528699876703
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.057130297017528,
      "ece_score": 88.5739405964944,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.5739405964944,
      "score_before_time_limit": 88.5739405964944,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.1123272461650836,
      "ece": 0.06861261261261234,
      "n": 375,
      "score": 86.27747747747753,
      "score_before_time_limit": 86.27747747747753,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9226666666666666,
      "interval90_width": 0.2508888888888888,
      "n": 375,
      "nrps": 0.07405460121549541,
      "score": 86.43271776381279,
      "score_before_time_limit": 86.43271776381279,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09864552175412443
     }
    },
    "score": 87.09471194592824
   },
   "capability": 78.90688322585615,
   "composite_ci95": {
    "A": [
     67.8354519561397,
     70.62112178058348
    ],
    "B": [
     67.46288814081362,
     71.39156955703
    ],
    "C": [
     67.8354519561397,
     70.62112178058348
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: exact base google/gemma-4-31B-it (served as Google's unchanged google/gemma-4-31B-it-qat-w4a16-ct@52f3f65b + own LoRA). OpenRouter google/gemma-4-31b-it list price USD 0.09/M input, USD 0.34/M output (listed since Apr 2026; same reference as the published Jeff, deck31b and Quyet-1.0-Large rows), read 2026-10-12T00:13:38Z from https://openrouter.ai/api/v1/models x the service's own measured usage tokens; estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.043874 per 1,000 decisions (0.679x the official cost cap USD 0.064595), against USD 0.049211 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04921078
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04387442424242424
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-31B-trained",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline read-only container (--network none); author's own service (service/serve.sh -> launch.py -> server.py on vLLM 0.31.0, pinned digest image), unchanged defaults Paid fast-lane order 147c5e18 (public). Own LoRA adapter (rank 16) on Google's official W4A16 Gemma 4 31B QAT checkpoint, applied unmerged by vLLM; max_model_len 6656, longer inputs are shortened by the service's documented extraction rule (author-tested only on H200; measured on H100 80GB with unchanged settings).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.5520469583762,
    "I_sealed": 68.88606205319194,
    "base": 70.71905450578407,
    "excess": -6.173541865171451,
    "gap": 3.66598490518426,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 80.81042171537538,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 73.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 63.24668534332811,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 70.41765078820325,
       "hard": 69.57893658072337,
       "judge": 60.72068725487829,
       "standard": 50.785697278774734
      }
     },
     "sealed|choice": {
      "cc": 80.73173886349481,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 78.142708453799,
       "judge": 78.84427032321254,
       "standard": 81.96582999367222
      }
     },
     "sealed|noul": {
      "cc": 74.61818469744357,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 67.27272727272727,
       "judge": 81.53846153846153,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 51.30826259863744,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.15351391347656,
       "hard": 51.34670276898963,
       "judge": 46.83933293506929,
       "standard": 48.51215109586572
      }
     }
    }
   },
   "jevbench_score": 69.591292225436,
   "key": "metask_jev_rain_31b_trained",
   "last_measured_on": "2026-10-12",
   "licence": "Weights: Apache-2.0 per model card (base Gemma 4 under the Gemma 4 licence); service code (service/) MIT",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "MeTask-Rain/metask-jev-rain-31B-trained@da56acc0bb607cc214c184ad9f1e9231a0d63dee",
   "noul_decisive": {
    "acc_among_decisive": 0.8773333333333333,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.09471194592824,
     "cost": 50.73365709012802,
     "intelligence": 70.71905450578407,
     "speed": 83.17884674032656
    },
    "board_release": "v1.6.43",
    "capability": 78.90688322585615,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 8,
    "capability_rank": 15,
    "composite_ci95": {
     "A": [
      67.97744875057778,
      70.7750334864619
     ],
     "B": [
      67.57519316040789,
      71.5173477277133
     ],
     "C": [
      67.97744875057778,
      70.7750334864619
     ]
    },
    "jevbench_score": 69.74074308829535,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 12,
    "rank": 28,
    "ranks": {
     "A": 28,
     "B": 21,
     "C": 26,
     "capability": 30
    },
    "scores": {
     "A": 69.74074308829535,
     "B": 69.93423396579702,
     "C": 69.74074308829535
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5687712740385905,
     "p50_s_raw": 0.20938563701929525,
     "p95_s_adjusted": 0.8456247233552858,
     "p95_s_raw": 0.3478123616776429
    },
    "views": {
     "Balanced 33:33:33": 65.39719257145047,
     "Emphasis on Accuracy 60:20:20": 67.42683688195376,
     "Emphasis on Cost 20:20:60": 58.62002091885354,
     "Emphasis on Speed 20:60:20": 71.51223704651308,
     "Intelligence only": 70.71905450578407
    }
   },
   "rank": 31,
   "ranked": true,
   "ranks": {
    "A": 31,
    "B": 23,
    "C": 27,
    "capability": 35
   },
   "repo": "https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.591292225436,
    "B": 69.81395795243358,
    "C": 69.591292225436
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6160417439881712,
    "p50_s_raw": 0.2330208719940856,
    "p95_s_adjusted": 0.9481153076398185,
    "p95_s_raw": 0.39905765381990926
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 63.16886779201999,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 14.506603376059957
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.16886779201999
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 64.53799979415957,
       "cc_by_type": {
        "choice": 77.7306468716861,
        "noul": 60.00000000000001,
        "score": 55.88335251079264
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 64.53799979415957
      },
      "composite_gen": {
       "accuracy": 0.8811881188118812,
       "cc": 83.81438823065723,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 62.102604619704316
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 83.81438823065723
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.04066368360208,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 98.12199105080622
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.04066368360208
      },
      "jaggedness": {
       "accuracy": 0.6666666666666666,
       "cc": 49.85525524192105,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 51.35135135135136,
        "score": 35.750506770946934
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.85525524192105
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.7793054809374,
       "cc_by_type": {
        "choice": 93.66029761380645,
        "noul": 95.8762886597938,
        "score": 61.801330169211944
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.7793054809374
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 9.832941925502283,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0.0,
        "score": 19.821406421668144
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 9.832941925502283
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 54.59598157228413,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75.0,
        "score": 25.901346778708078
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 54.59598157228413
      },
      "multilingual": {
       "accuracy": 0.7963636363636364,
       "cc": 68.20002344704231,
       "cc_by_type": {
        "choice": 75.54613628953375,
        "noul": 69.44444444444444,
        "score": 59.60948960714872
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.20002344704231
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 68.5323347023584,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 52.263670773741836
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 68.5323347023584
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 81.60522267243225,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 59.83068303231177
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.60522267243225
      },
      "usecase": {
       "accuracy": 0.7363636363636363,
       "cc": 64.54982362341306,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 70.83333333333333,
        "score": 46.100451262396035
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.54982362341306
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 36.07646009918423,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": -3.675381607209216
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 36.07646009918423
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 73.84108691654622,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 42.260587938578766
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 73.84108691654622
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 90.42477813573423,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 87.20353794702571
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.42477813573423
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.579631749179,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.59178165349604
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.579631749179
      },
      "en": {
       "accuracy": 0.8227628149435273,
       "cc": 71.3569930395452,
       "cc_by_type": {
        "choice": 83.0853907079484,
        "noul": 75.53956834532374,
        "score": 55.44602006536346
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.3569930395452
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 69.16556628426598,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 27.07255529651733
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.16556628426598
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.9244647500369,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 99.56027949601237
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.9244647500369
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 93.75128961226879,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 81.25386883680636
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 93.75128961226879
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 73.31989019858385,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0,
        "score": 19.959670595751543
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 73.31989019858385
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 56.92598694485033,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -15.192421082830165
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 56.92598694485033
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 46.34982921310412,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 80.0,
        "score": 1.9406711182639036
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 46.34982921310412
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 60.502271657899314,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 81.50681497369794
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 60.502271657899314
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.73180568607792,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.86208372490044
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.73180568607792
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 8.642558761906725,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 11.209061567105461
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 8.642558761906725
      },
      "no": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 100.0
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 63.392816051625765,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 57.79588361827128
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 63.392816051625765
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 67.80504865797501,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 56.44991321022436
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 67.80504865797501
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 46.19054683365891,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 29.63547028821076
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 46.19054683365891
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 68.75012962289753,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 27.989519303475184
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 68.75012962289753
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 61.68878465176133,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 49.582482987542065
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 61.68878465176133
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 69.21114832796756,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 7.63344498390266
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 69.21114832796756
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8573333333333333,
       "cc": 81.57255209628963,
       "cc_by_type": {
        "choice": 81.57255209628963
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.57255209628963
      },
      "noul": {
       "accuracy": 0.8773333333333333,
       "cc": 75.46666666666665,
       "cc_by_type": {
        "noul": 75.46666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 75.46666666666665
      },
      "score": {
       "accuracy": 0.6746666666666666,
       "cc": 54.7684541832536,
       "cc_by_type": {
        "score": 54.7684541832536
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 54.7684541832536
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 48.45861881975295,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 25.0,
        "score": 36.375856459258834
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 48.45861881975295
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 76.75345303484892,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 30.26035910454675
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 76.75345303484892
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 69.6944643957582,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100.0,
        "score": 25.343555788900616
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 69.6944643957582
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -24.108890369968993,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -33.7301798818368
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 57.255300551831404,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 67.41807556853769
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 57.255300551831404
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 35.41674609079076,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -35.25919568989187
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 35.41674609079076
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 56.61757692065063,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 71.42857142857142,
        "score": 27.835924039262792
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.61757692065063
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 43.298609533332666,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0.0,
        "score": 61.2547136174196
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 43.298609533332666
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 40.88581737462815,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 25.15745212388444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 40.88581737462815
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 70.06051887452556,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100.0,
        "score": 35.877488101092766
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 70.06051887452556
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 64.06806981529785,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 14.426431668115768
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 64.06806981529785
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 49.78201560348273,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 46.1202403588353
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 49.78201560348273
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 72.7024042480826,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 58.107212744247796
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.7024042480826
      },
      "model_routing": {
       "accuracy": 0.9032258064516129,
       "cc": 80.21287919965458,
       "cc_by_type": {
        "choice": 91.81818181818183,
        "noul": 100.0,
        "score": 48.82045578078191
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.21287919965458
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 48.98671618683705,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 69.9927781190524
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 48.98671618683705
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 78.10700850764788,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100.0,
        "score": 67.74720936695476
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 78.10700850764788
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 56.67924976273154,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100.0,
        "score": -0.40560046549995565
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 56.67924976273154
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 45.24701491441865,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 45.88597227948784
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 45.24701491441865
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.61559815097014,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.84679445291042
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.61559815097014
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.90626522739525,
      88.6737670940263
     ],
     "I": [
      65.93306472259675,
      74.78359254040711
     ],
     "capability": [
      74.80944444127567,
      81.28198974405463
     ],
     "composite": {
      "A": [
       67.8354519561397,
       70.62112178058348
      ],
      "B": [
       67.46288814081362,
       71.39156955703
      ],
      "C": [
       67.8354519561397,
       70.62112178058348
      ]
     },
     "seed": 48
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.09471194592824,
     "capability": 78.90688322585615,
     "intelligence": 70.71905450578407
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8573333333333333,
      "calibration": 88.5739405964944,
      "cc": 81.57255209628963,
      "cc_by_type": {
       "choice": 81.57255209628963
      },
      "cc_tiered": 80.7710802894351,
      "cc_tiered_open": 80.81042171537538,
      "cc_tiered_sealed": 80.73173886349481,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.57255209628963
     },
     "noul": {
      "accuracy": 0.8773333333333333,
      "calibration": 86.27747747747753,
      "cc": 75.46666666666665,
      "cc_by_type": {
       "noul": 75.46666666666665
      },
      "cc_tiered": 74.10860925693434,
      "cc_tiered_open": 73.59903381642512,
      "cc_tiered_sealed": 74.61818469744357,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 75.46666666666665
     },
     "score": {
      "accuracy": 0.6746666666666666,
      "calibration": 86.43271776381279,
      "cc": 54.7684541832536,
      "cc_by_type": {
       "score": 54.7684541832536
      },
      "cc_tiered": 57.277473970982776,
      "cc_tiered_open": 63.24668534332811,
      "cc_tiered_sealed": 51.30826259863744,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.7684541832536
     }
    },
    "raw_on_A": {
     "calibration": 83.28617618857919,
     "capability": 76.3197068643228,
     "intelligence": 69.35323754006639
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.22206708454334,
    "Emphasis on Accuracy 60:20:20": 67.31502402762642,
    "Emphasis on Cost 20:20:60": 58.53549066472637,
    "Emphasis on Speed 20:60:20": 71.13627872356572,
    "Intelligence only": 70.71905450578407
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "MeTask (Hangzhou MeTask Technology Co., Ltd.)",
   "axes": {
    "calibration": 86.60202339920977,
    "cost": 49.27770156907625,
    "intelligence": 72.59243765084337,
    "speed": 73.27177592289428
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06613866866964949,
      "ece_score": 86.7722662660701,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.7722662660701,
      "score_before_time_limit": 86.7722662660701,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.1016786666666672,
      "ece": 0.06551999999999943,
      "n": 375,
      "score": 86.89600000000011,
      "score_before_time_limit": 86.89600000000011,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.20799999999999996,
      "n": 375,
      "nrps": 0.06738985445027339,
      "score": 86.13780393155909,
      "score_before_time_limit": 86.13780393155909,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10492703345927233
     }
    },
    "score": 86.60202339920977
   },
   "capability": 79.59723052502656,
   "composite_ci95": {
    "A": [
     64.26152478726674,
     66.51834374104213
    ],
    "B": [
     64.5941314338483,
     67.88120664117577
    ],
    "C": [
     64.26152478726674,
     66.51834374104213
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: exact base google/gemma-4-31B-it, OpenRouter google/gemma-4-31b-it list price USD 0.09/M input, USD 0.34/M output (the same reference as the published metask_jev_rain_31b_trained row; snapshot read 2026-10-13T12:10:25Z by the order's public-docs receipt) x the service's own measured usage tokens (840,049 in / 17,397 out); estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.049062 per 1,000 decisions (0.760x the official cost cap USD 0.064595), against USD 0.054346 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05434626
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04906177777777778
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-31B-trained service v3",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX PRO 6000 96 GB; the reviewed recipe needs 96 GB), offline read-only container (--network none, verified by docker inspect before the input upload); the author's own service at revision 2feff076 (service v3), unchanged defaults. Paid public fast-lane order 4d9fef10. RE-SUBMISSION, DISCLOSED: the same LoRA weights as the published metask_jev_rain_31b_trained row (order 147c5e18, v1.6.10) with a changed service/ layer (e.g. its yes/no constant); those changes may have been tuned on that row's published v1.6.10 aggregates (custodian #18945). Both rows are shown; no third same-weights row is admitted on this draw. Weights file shas: model 1b9b1d62..., adapter 2b956899..., code tree 6a1234ee...",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.52301075937709,
    "I_sealed": 72.66186454230964,
    "base": 72.59243765084337,
    "excess": -9.978380553288268,
    "gap": -0.13885378293255712,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 80.81042171537538,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 73.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 63.15957674633078,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.16584682233966,
       "hard": 69.17688425857045,
       "judge": 60.01699079087648,
       "standard": 50.83570561702846
      }
     },
     "sealed|choice": {
      "cc": 82.58817768397486,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 80.2243552677229,
       "judge": 79.72575905974536,
       "standard": 85.76249736342545
      }
     },
     "sealed|noul": {
      "cc": 76.53948727546258,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 69.0909090909091,
       "judge": 81.53846153846153,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 58.85792866749151,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 69.80503007296836,
       "hard": 56.95507418481911,
       "judge": 61.45515272353761,
       "standard": 53.2942508460287
      }
     }
    }
   },
   "jevbench_score": 65.55909153695762,
   "key": "metask_jev_rain_31b_trained_v3",
   "last_measured_on": "2026-10-13",
   "licence": "Weights: Apache-2.0 per model card (base Gemma 4 under the Gemma 4 licence); service code (service/) MIT",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "MeTask-Rain/metask-jev-rain-31B-trained@2feff076f07711fe2fefe7a6fdcedfe4132b773e",
   "noul_decisive": {
    "acc_among_decisive": 0.8853333333333333,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.60202339920977,
     "cost": 49.27770156907625,
     "intelligence": 72.59243765084337,
     "speed": 84.66561996644248
    },
    "board_release": "v1.6.43",
    "capability": 79.59723052502656,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 6,
    "capability_rank": 13,
    "composite_ci95": {
     "A": [
      66.27480907834259,
      68.67789744912794
     ],
     "B": [
      66.21154989482208,
      69.66970736124362
     ],
     "C": [
      66.27480907834259,
      68.67789744912794
     ]
    },
    "jevbench_score": 67.6558274201148,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": 21,
    "rank": 41,
    "ranks": {
     "A": 41,
     "B": 27,
     "C": 34,
     "capability": 25
    },
    "scores": {
     "A": 67.6558274201148,
     "B": 68.20807250966563,
     "C": 67.6558274201148
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5115797939710319,
     "p50_s_raw": 0.18078989698551595,
     "p95_s_adjusted": 0.6676127476617694,
     "p95_s_raw": 0.2588063738308847
    },
    "views": {
     "Balanced 33:33:33": 63.51261802145979,
     "Emphasis on Accuracy 60:20:20": 66.13810967705214,
     "Emphasis on Cost 20:20:60": 56.167442462815025,
     "Emphasis on Speed 20:60:20": 69.87668805746007,
     "Intelligence only": 70.5102505244786
    }
   },
   "rank": 54,
   "ranked": true,
   "ranks": {
    "A": 54,
    "B": 41,
    "C": 40,
    "capability": 31
   },
   "repo": "https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 65.55909153695762,
    "B": 66.49290453419427,
    "C": 65.55909153695762
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5195755619555712,
    "p50_s_raw": 0.1847877809777856,
    "p95_s_adjusted": 9.06094911987893,
    "p95_s_raw": 4.455474559939465
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 89.03475546368588,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 83.77093305772432
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 89.03475546368588
      },
      "calibration_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 68.68645561782115,
       "cc_by_type": {
        "choice": 77.7306468716861,
        "noul": 68.0,
        "score": 60.328719981777354
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 68.68645561782115
      },
      "composite_gen": {
       "accuracy": 0.8910891089108911,
       "cc": 84.77793164693051,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 64.99323486852417
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 84.77793164693051
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.04372227473641,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 98.13116682420925
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.04372227473641
      },
      "jaggedness": {
       "accuracy": 0.7,
       "cc": 55.63506291104909,
       "cc_by_type": {
        "choice": 69.68238691049086,
        "noul": 56.75675675675676,
        "score": 40.46604506589963
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 55.63506291104909
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.60538459908456,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 95.8762886597938,
        "score": 60.48710472537918
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.60538459908456
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 8.720258887926093,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0.0,
        "score": 16.483357308939574
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 8.720258887926093
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 62.009380987023235,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75.0,
        "score": 48.14154502292535
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 62.009380987023235
      },
      "multilingual": {
       "accuracy": 0.8145454545454546,
       "cc": 70.77751409077383,
       "cc_by_type": {
        "choice": 77.50244538637104,
        "noul": 72.22222222222223,
        "score": 62.60787466372824
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.77751409077383
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 71.91781286399173,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 62.420105258641826
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 71.91781286399173
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 82.80777431471158,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 63.43833795914975
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.80777431471158
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 68.49729252167786,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 70.83333333333333,
        "score": 57.942857957190455
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 68.49729252167786
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 43.24855438365077,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": 17.84090124619041
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 43.24855438365077
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 74.01835450650829,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 42.79239070846494
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 74.01835450650829
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 81.83435068683548,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 61.432255600329455
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 81.83435068683548
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.60222006784046,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.65954660948044
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.60222006784046
      },
      "en": {
       "accuracy": 0.8410078192875761,
       "cc": 74.40408858051423,
       "cc_by_type": {
        "choice": 84.40341221122515,
        "noul": 76.97841726618707,
        "score": 61.83043626413047
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.40408858051423
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 68.2680286109703,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 24.379942276630317
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.2680286109703
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.93547668805367,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 99.59331531006269
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.93547668805367
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 93.7369976649519,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 81.2109929948557
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 93.7369976649519
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 80.07784582404595,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0,
        "score": 40.23353747213787
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 80.07784582404595
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 54.53760923768931,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -22.357554204313203
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 54.53760923768931
      },
      "ja": {
       "accuracy": 0.84,
       "cc": 64.88034315630443,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 80.0,
        "score": 28.938090628563796
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 64.88034315630443
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 60.26417092788616,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 80.79251278365848
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 60.26417092788616
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.93609912586727,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.80829737760182
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.93609912586727
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 8.53215978329243,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 10.877864631262579
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 8.53215978329243
      },
      "no": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 100.0
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 63.14426530245589,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 57.050231370761686
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 63.14426530245589
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 74.38219774143825,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 65.57440701335422
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.38219774143825
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 46.19054710826382,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 29.635471112025513
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 46.19054710826382
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 69.0713909756119,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 28.95330336161831
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 69.0713909756119
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 61.68232554763859,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 49.56310567517384
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 61.68232554763859
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 69.21240428354913,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 7.637212850647435
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 69.21240428354913
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8693333333333333,
       "cc": 83.12252434987275,
       "cc_by_type": {
        "choice": 83.12252434987275
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.12252434987275
      },
      "noul": {
       "accuracy": 0.8853333333333333,
       "cc": 77.06666666666666,
       "cc_by_type": {
        "noul": 77.06666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 77.06666666666666
      },
      "score": {
       "accuracy": 0.7173333333333334,
       "cc": 60.018133778302385,
       "cc_by_type": {
        "score": 60.018133778302385
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 60.018133778302385
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 49.15314922895124,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 25.0,
        "score": 38.45944768685374
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 49.15314922895124
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 80.079808393905,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 40.239425181714985
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.079808393905
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 84.23287684117527,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 85.21895572677785
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 84.23287684117527
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -20.319424506293327,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -22.361782290809806
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 57.24850688656365,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 67.39769457273441
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 57.24850688656365
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 30.523095261874122,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -49.940148176641785
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 30.523095261874122
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 60.33576603025934,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 71.42857142857142,
        "score": 38.99049136808895
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 60.33576603025934
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 46.327230880637806,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0.0,
        "score": 70.340577659335
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 46.327230880637806
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 54.927785267620116,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 67.28335580286033
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 54.927785267620116
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 78.22201831947774,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100.0,
        "score": 47.51402069719127
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 78.22201831947774
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 63.998217861086424,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 14.216875805481488
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.998217861086424
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 51.14449916437249,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 50.207691041504575
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 51.14449916437249
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 75.39671275196183,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 66.19013825588543
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 75.39671275196183
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 80.45531443691877,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100.0,
        "score": 48.37893032374332
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.45531443691877
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 56.46101690387395,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 92.4156802701631
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 56.46101690387395
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 82.04193370234395,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100.0,
        "score": 79.55198495104301
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 82.04193370234395
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 56.88699086990223,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100.0,
        "score": 0.21762285601210074
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 56.88699086990223
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 47.859636984518865,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 53.7238384897885
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 47.859636984518865
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.6261843392683,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.87855301780489
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.6261843392683
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.1062803062735,
      89.00397681987438
     ],
     "I": [
      67.96927371270253,
      76.63101848971161
     ],
     "capability": [
      76.2469833472818,
      82.24205839797754
     ],
     "composite": {
      "A": [
       64.26152478726674,
       66.51834374104213
      ],
      "B": [
       64.5941314338483,
       67.88120664117577
      ],
      "C": [
       64.26152478726674,
       66.51834374104213
      ]
     },
     "seed": 49
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.60202339920977,
     "capability": 79.59723052502656,
     "intelligence": 72.59243765084337
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8693333333333333,
      "calibration": 86.7722662660701,
      "cc": 83.12252434987275,
      "cc_by_type": {
       "choice": 83.12252434987275
      },
      "cc_tiered": 81.69929969967512,
      "cc_tiered_open": 80.81042171537538,
      "cc_tiered_sealed": 82.58817768397486,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.12252434987275
     },
     "noul": {
      "accuracy": 0.8853333333333333,
      "calibration": 86.89600000000011,
      "cc": 77.06666666666666,
      "cc_by_type": {
       "noul": 77.06666666666666
      },
      "cc_tiered": 75.06926054594385,
      "cc_tiered_open": 73.59903381642512,
      "cc_tiered_sealed": 76.53948727546258,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 77.06666666666666
     },
     "score": {
      "accuracy": 0.7173333333333334,
      "calibration": 86.13780393155909,
      "cc": 60.018133778302385,
      "cc_by_type": {
       "score": 60.018133778302385
      },
      "cc_tiered": 61.00875270691114,
      "cc_tiered_open": 63.15957674633078,
      "cc_tiered_sealed": 58.85792866749151,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.018133778302385
     }
    },
    "raw_on_A": {
     "calibration": 81.92265838998793,
     "capability": 76.06423959692178,
     "intelligence": 70.20582080385563
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 61.067961948452634,
    "Emphasis on Accuracy 60:20:20": 64.52423350241737,
    "Emphasis on Cost 20:20:60": 54.99919048625603,
    "Emphasis on Speed 20:60:20": 64.7439583643371,
    "Intelligence only": 70.5102505244786
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": true,
   "author": "Microsoft",
   "axes": {
    "calibration": 84.84355738336595,
    "cost": 62.19876038790379,
    "intelligence": 59.226098571054024,
    "speed": 88.66907977364731
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1108223199674765,
      "ece_score": 77.8355360065047,
      "mean_tvd": null,
      "n": 296,
      "n_ece_hard": 128,
      "n_tvd": 0,
      "score": 77.8355360065047,
      "score_before_time_limit": 77.8355360065047,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.22972972972972971,
      "brier": 0.08789565767934152,
      "ece": 0.06454285728875721,
      "n": 148,
      "score": 87.09142854224856,
      "score_before_time_limit": 87.09142854224856,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9194630872483222,
      "interval90_width": 0.1784116331096197,
      "n": 149,
      "nrps": 0.06306645857440242,
      "score": 87.1325461475575,
      "score_before_time_limit": 87.1325461475575,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09714130923722374
     }
    },
    "score": 84.01983689877025
   },
   "canonical_system_key": "microsoft-decision-1",
   "capability": 72.03482797720999,
   "composite_ci95": {
    "A": [
     59.165096184175106,
     74.53230995103505
    ],
    "B": [
     54.902945001197075,
     73.17537269212036
    ],
    "C": [
     41.08687235012159,
     74.53230995103505
    ]
   },
   "cost": {
    "basis": "LEAD DECISION r3: USD 0.042/M input and USD 0/M output, basis of the live Microsoft-Decision-1 (Azure Foundry) row on https://benchmarkheaven.com/jev-models/api. Reported usage across all 600 decisions, including seven HTTP422 refusals retained in denominator; frozen parent all-item cost basis. Invoice, warmup and refusal charges unknown.",
    "common_basis": null,
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01819874
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Microsoft-Decision-1 (Azure Foundry) — A2 re-run",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted Azure Foundry API; latency measured from Sandy (Germany). Seven of 600 HTTP 422 responses scored wrong. Four self-hosted systems re-run on H200, versus H100 parents; wide equating interval. MeTask 31B and Torchcast not completed after four failed pod attempts; see measurement notes.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.00640482707561,
    "I_sealed": 46.29756060035449,
    "base": 56.65198271371505,
    "excess": 3.8882152491360813,
    "gap": 20.708844226721126,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 82.78833309663467,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 73.57659641396666,
       "judge": 84.17930125247197,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 54.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 63.288852399084945,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.860607370724196,
       "hard": 67.50339329867514,
       "judge": 74.36852911363773,
       "standard": 43.45437804225577
      }
     },
     "sealed|choice": {
      "cc": 63.30695607434754,
      "n": {
       "easy": 15,
       "hard": 64,
       "judge": 35,
       "standard": 36
      },
      "tiers": {
       "easy": 35.48387096774193,
       "hard": 52.901023890784984,
       "judge": 76.86375321336762,
       "standard": 77.69516728624535
      }
     },
     "sealed|noul": {
      "cc": 23.62820512820513,
      "n": {
       "easy": 13,
       "hard": 32,
       "judge": 15,
       "standard": 15
      },
      "tiers": {
       "easy": 84.61538461538463,
       "hard": 6.25,
       "judge": -6.666666666666665,
       "standard": 73.33333333333334
      }
     },
     "sealed|score": {
      "cc": 51.95752059851082,
      "n": {
       "easy": 9,
       "hard": 33,
       "judge": 20,
       "standard": 13
      },
      "tiers": {
       "easy": 76.44302715293294,
       "hard": 49.562049785657535,
       "judge": 44.71869741001873,
       "standard": 55.363943729744456
      }
     }
    }
   },
   "jevbench_score": 71.39513496315271,
   "key": "microsoft-decision-1-a2-20261012",
   "last_measured_on": "2026-10-12",
   "listing": "preliminary",
   "measured_in": "v1.6.14",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "not_ranked_because": "A2 supplement measurement with an incomplete equating pool (four of six); excluded from every ranking.",
   "planned_weight_denominator": 1,
   "preliminary": {
    "equating_pool_size": 4,
    "equating_pool_target": 6,
    "label": "PRELIMINARY — API supplement A2 (300 sealed + 300 public), not the S1200 full set; equating pool 4 of 6",
    "measured_on": "2026-10-12",
    "n_items": 600,
    "n_ok": 593,
    "scope": "API supplement A2"
   },
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.15566582345811,
     "cost": 62.19876038790379,
     "intelligence": 59.226098571054024,
     "speed": 88.69804989142763
    },
    "board_release": "v1.6.43",
    "capability": 72.19088219725606,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      60.33591615493133,
      74.53742586189195
     ],
     "B": [
      56.31152174249009,
      73.1804420130518
     ],
     "C": [
      41.89994177425786,
      74.53742586189195
     ]
    },
    "jevbench_score": 71.4549280926844,
    "measured_in": "v1.6.14",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 71.4549280926844,
     "B": 68.62118813181577,
     "C": 69.62351351359915
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.3374342759998399,
     "p50_s_raw": 0.3374342759998399,
     "p95_s_adjusted": 0.3999501053016501,
     "p95_s_raw": 0.3999501053016501
    },
    "views": {
     "Balanced 33:33:33": 67.81784570375109,
     "Emphasis on Accuracy 60:20:20": 64.0984217053166,
     "Emphasis on Cost 20:20:60": 65.4526287582965,
     "Emphasis on Speed 20:60:20": 74.86761060072513,
     "Intelligence only": 59.22609857105403
    }
   },
   "provider": "microsoft",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.39513496315271,
    "B": 68.57706385643915,
    "C": 69.5652529027065
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.33810210000046936,
    "p50_s_raw": 0.33810210000046936,
    "p95_s_adjusted": 0.4018316626976227,
    "p95_s_raw": 0.4018316626976227
   },
   "status": {
    "answered_ok": 593,
    "missing": 0,
    "rows": 600,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": true,
    "api_subset_tag": "A2",
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 100.0
      },
      "calibration_gen": {
       "accuracy": 0.7692307692307693,
       "cc": 63.29111148533698,
       "cc_by_type": {
        "choice": 60.8695652173913,
        "noul": 75.0,
        "score": 54.003769238619604
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 63.29111148533698
      },
      "composite_gen": {
       "accuracy": 0.7741935483870968,
       "cc": 55.04557875346427,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 45.136736260392794
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 10,
        "noul": 10,
        "score": 11
       },
       "score": 55.04557875346427
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 99.45663234640188,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.36989703920564
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 6
       },
       "score": 99.45663234640188
      },
      "jaggedness": {
       "accuracy": 0.6,
       "cc": 36.18096361245757,
       "cc_by_type": {
        "choice": 72.78911564625851,
        "noul": -28.57142857142857,
        "score": 64.32520376254277
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 20,
        "noul": 14,
        "score": 11
       },
       "score": 36.18096361245757
      },
      "llm_router": {
       "accuracy": 0.8155339805825242,
       "cc": 77.29773373392734,
       "cc_by_type": {
        "choice": 66.38655462184875,
        "noul": 100.0,
        "score": 65.50664657993329
       },
       "low_n": false,
       "n": 103,
       "n_by_type": {
        "choice": 50,
        "noul": 30,
        "score": 23
       },
       "score": 77.29773373392734
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -110.3825136612022,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100.0,
        "score": -200.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 1
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.7405660377358491,
       "cc": 56.84729346177679,
       "cc_by_type": {
        "choice": 67.37084291989123,
        "noul": 39.99999999999999,
        "score": 63.17103746543915
       },
       "low_n": false,
       "n": 212,
       "n_by_type": {
        "choice": 107,
        "noul": 40,
        "score": 65
       },
       "score": 56.84729346177679
      },
      "negation_pair_gen": {
       "accuracy": 0.8,
       "cc": 60.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 10,
        "noul": 10
       },
       "score": 60.0
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 21
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 1.0,
       "cc": 97.86434741302423,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 93.59304223907267
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 4
       },
       "score": 97.86434741302423
      },
      "usecase": {
       "accuracy": 0.7058823529411765,
       "cc": 37.19296114334664,
       "cc_by_type": {
        "choice": 67.7891654465593,
        "noul": 10.000000000000009,
        "score": 33.78971798348063
       },
       "low_n": false,
       "n": 85,
       "n_by_type": {
        "choice": 44,
        "noul": 20,
        "score": 21
       },
       "score": 37.19296114334664
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8888888888888888,
       "cc": 92.43674893442142,
       "cc_by_type": {
        "choice": 77.77777777777779,
        "noul": 100.0,
        "score": 99.53246902548645
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 1
       },
       "score": 92.43674893442142
      },
      "cs": {
       "accuracy": 0.625,
       "cc": 66.1654135338346,
       "cc_by_type": {
        "choice": 32.330827067669176,
        "noul": 100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 6,
        "noul": 2
       },
       "score": 66.1654135338346
      },
      "da": {
       "accuracy": 0.6666666666666666,
       "cc": 29.010593471104,
       "cc_by_type": {
        "choice": 80.83067092651757,
        "noul": 0.0,
        "score": 6.2011094867944205
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 1
       },
       "score": 29.010593471104
      },
      "de": {
       "accuracy": 0.9230769230769231,
       "cc": 90.62759668123668,
       "cc_by_type": {
        "choice": 74.64788732394366,
        "noul": 100.0,
        "score": 97.23490271976637
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 4
       },
       "score": 90.62759668123668
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": -21.42857142857143,
       "cc_by_type": {
        "choice": 57.14285714285714,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 3,
        "noul": 3
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.7753424657534247,
       "cc": 58.59078985584421,
       "cc_by_type": {
        "choice": 76.21638426861493,
        "noul": 42.574257425742566,
        "score": 56.981727873175124
       },
       "low_n": false,
       "n": 365,
       "n_by_type": {
        "choice": 185,
        "noul": 101,
        "score": 79
       },
       "score": 58.59078985584421
      },
      "es": {
       "accuracy": 0.9166666666666666,
       "cc": 70.0797903785218,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 10.239371135565412
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 1
       },
       "score": 70.0797903785218
      },
      "fi": {
       "accuracy": 1.0,
       "cc": 94.53482291929028,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 83.60446875787086
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 2,
        "noul": 2,
        "score": 6
       },
       "score": 94.53482291929028
      },
      "fr": {
       "accuracy": 0.75,
       "cc": 83.59037174371464,
       "cc_by_type": {
        "choice": 51.99999999999999,
        "noul": 100.0,
        "score": 98.77111523114391
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 1
       },
       "score": 83.59037174371464
      },
      "hi": {
       "accuracy": 1.0,
       "cc": 92.81199164662956,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 78.43597493988868
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 1
       },
       "score": 92.81199164662956
      },
      "id": {
       "accuracy": 0.7272727272727273,
       "cc": 70.7560754894387,
       "cc_by_type": {
        "choice": 73.09417040358746,
        "score": 68.41798057528993
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "score": 6
       },
       "score": 70.7560754894387
      },
      "it": {
       "accuracy": 0.7272727272727273,
       "cc": 80.04349357771646,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 40.13048073314939
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 2,
        "noul": 2,
        "score": 7
       },
       "score": 80.04349357771646
      },
      "ja": {
       "accuracy": 0.6363636363636364,
       "cc": 47.61381032028706,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 50.0,
        "score": 91.20208669856609
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 4,
        "score": 3
       },
       "score": 47.61381032028706
      },
      "ko": {
       "accuracy": 0.8181818181818182,
       "cc": 77.78079668122632,
       "cc_by_type": {
        "choice": 100.0,
        "score": 55.561593362452655
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 3,
        "score": 8
       },
       "score": 77.78079668122632
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 11,
        "noul": 2
       },
       "score": 100.0
      },
      "nl": {
       "accuracy": 0.5714285714285714,
       "cc": 1.7832893135850607,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 5.349867940755182
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 1
       },
       "score": 1.7832893135850607
      },
      "no": {
       "accuracy": 0.6363636363636364,
       "cc": 39.253029666918,
       "cc_by_type": {
        "choice": 14.893617021276594,
        "noul": 33.33333333333333,
        "score": 69.53213864614409
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 3,
        "noul": 3,
        "score": 5
       },
       "score": 39.253029666918
      },
      "pl": {
       "accuracy": 0.7,
       "cc": 52.957577540994386,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 50.0,
        "score": 40.901202373872835
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 8
       },
       "score": 52.957577540994386
      },
      "pt": {
       "accuracy": 0.6923076923076923,
       "cc": 74.25511482979587,
       "cc_by_type": {
        "choice": 26.530612244897956,
        "noul": 100.0,
        "score": 96.23473224448966
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 2
       },
       "score": 74.25511482979587
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 3.3459357774096503,
       "cc_by_type": {
        "choice": 33.70165745856353,
        "noul": -100.0,
        "score": 76.33614987366542
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 6
       },
       "score": 3.3459357774096503
      },
      "tr": {
       "accuracy": 0.7142857142857143,
       "cc": 77.61194029850746,
       "cc_by_type": {
        "choice": 55.22388059701492,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 77.61194029850746
      },
      "uk": {
       "accuracy": 0.5384615384615384,
       "cc": -2.908915160021811,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 35.71769896437901
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 3,
        "noul": 1,
        "score": 9
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.8333333333333334,
       "cc": 89.63050869746054,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 100.0,
        "score": 98.30329079826399
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 1
       },
       "score": 89.63050869746054
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7933333333333333,
       "cc": 73.01414581066375,
       "cc_by_type": {
        "choice": 73.01414581066375
       },
       "low_n": false,
       "n": 300,
       "n_by_type": {
        "choice": 300
       },
       "score": 73.01414581066375
      },
      "noul": {
       "accuracy": 0.7266666666666667,
       "cc": 45.333333333333336,
       "cc_by_type": {
        "noul": 45.333333333333336
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "noul": 150
       },
       "score": 45.333333333333336
      },
      "score": {
       "accuracy": 0.7466666666666667,
       "cc": 59.17272659737858,
       "cc_by_type": {
        "score": 59.17272659737858
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "score": 150
       },
       "score": 59.17272659737858
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6153846153846154,
       "cc": 22.655509813036304,
       "cc_by_type": {
        "choice": 73.7991266375546,
        "noul": -33.333333333333336,
        "score": 27.50073613488765
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 5
       },
       "score": 22.655509813036304
      },
      "code_linting": {
       "accuracy": 0.5555555555555556,
       "cc": 30.103576043627953,
       "cc_by_type": {
        "choice": 56.83453237410072,
        "noul": 0.0,
        "score": 33.47619575678314
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.103576043627953
      },
      "customer_support": {
       "accuracy": 0.875,
       "cc": 62.22727451527351,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 86.68182354582052
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 1
       },
       "score": 62.22727451527351
      },
      "demand_forecasting": {
       "accuracy": 0.2857142857142857,
       "cc": -25.73258663586017,
       "cc_by_type": {
        "choice": 16.89750692520776,
        "noul": -33.333333333333336,
        "score": -60.76193349945493
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6875,
       "cc": 37.128494536040996,
       "cc_by_type": {
        "choice": 51.61290322580645,
        "noul": -33.333333333333336,
        "score": 93.10591371564988
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 37.128494536040996
      },
      "feature_extraction": {
       "accuracy": 0.875,
       "cc": 79.10347860874892,
       "cc_by_type": {
        "choice": 85.94847775175644,
        "noul": 100.0,
        "score": 51.361958074490275
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 1,
        "score": 6
       },
       "score": 79.10347860874892
      },
      "financial_crime": {
       "accuracy": 0.8666666666666667,
       "cc": 86.22232602398712,
       "cc_by_type": {
        "choice": 80.70739549839227,
        "noul": 100.0,
        "score": 77.95958257356911
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 7,
        "noul": 2,
        "score": 6
       },
       "score": 86.22232602398712
      },
      "gaming": {
       "accuracy": 0.5333333333333333,
       "cc": 12.129649732040141,
       "cc_by_type": {
        "choice": 45.454545454545446,
        "noul": -33.333333333333336,
        "score": 24.267737074908315
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 7
       },
       "score": 12.129649732040141
      },
      "insurance_claims": {
       "accuracy": 0.5454545454545454,
       "cc": 11.15157429187248,
       "cc_by_type": {
        "choice": 43.573667711598745,
        "noul": 0.0,
        "score": -10.118944835981303
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 7,
        "noul": 2,
        "score": 2
       },
       "score": 11.15157429187248
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.92165664860643,
       "cc_by_type": {
        "choice": 56.83453237410071,
        "noul": 50.0,
        "score": 90.93043757171858
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 65.92165664860643
      },
      "lead_generation": {
       "accuracy": 0.5833333333333334,
       "cc": 32.27917770928797,
       "cc_by_type": {
        "choice": 52.56916996047431,
        "noul": 0.0,
        "score": 44.2683631673896
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 2
       },
       "score": 32.27917770928797
      },
      "legal_compliance": {
       "accuracy": 1.0,
       "cc": 94.12770453740406,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 82.38311361221218
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 4
       },
       "score": 94.12770453740406
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 87.81390014088669,
       "cc_by_type": {
        "choice": 78.14207650273225,
        "noul": 100.0,
        "score": 85.29962391992782
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 12,
        "noul": 1,
        "score": 5
       },
       "score": 87.81390014088669
      },
      "model_routing": {
       "accuracy": 0.8588235294117647,
       "cc": 81.49062517209573,
       "cc_by_type": {
        "choice": 79.13561847988078,
        "noul": 100.0,
        "score": 65.33625703640644
       },
       "low_n": false,
       "n": 85,
       "n_by_type": {
        "choice": 40,
        "noul": 24,
        "score": 21
       },
       "score": 81.49062517209573
      },
      "moderation_trust_safety": {
       "accuracy": 0.5833333333333334,
       "cc": 1.4715857588347916,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": -100.0,
        "score": 43.124434695859215
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 7,
        "noul": 1,
        "score": 4
       },
       "score": 1.4715857588347916
      },
      "other": {
       "accuracy": 0.4,
       "cc": 15.494864573175938,
       "cc_by_type": {
        "choice": 9.774436090225558,
        "score": 21.215293056126317
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 3,
        "score": 2
       },
       "score": 15.494864573175938
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 30.434646255259782,
       "cc_by_type": {
        "choice": 49.999999999999986,
        "noul": 0.0,
        "score": 41.30393876577936
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 4
       },
       "score": 30.434646255259782
      },
      "risk_assessment": {
       "accuracy": 0.75,
       "cc": 10.1937605340201,
       "cc_by_type": {
        "choice": 52.94117647058823,
        "noul": -100.0,
        "score": 77.64010513147205
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 5
       },
       "score": 10.1937605340201
      },
      "scientific_discovery": {
       "accuracy": 0.5714285714285714,
       "cc": 27.722363031181995,
       "cc_by_type": {
        "choice": 63.07692307692309,
        "noul": 0.0,
        "score": 20.09016601662289
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 7,
        "noul": 2,
        "score": 5
       },
       "score": 27.722363031181995
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.185292235786,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 94.55587670735798
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 4
       },
       "score": 98.185292235786
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.44693825231155,
      91.6174058639458
     ],
     "I": [
      47.46930943328842,
      68.3083574651847
     ],
     "capability": [
      64.10224318317934,
      77.56893471291383
     ],
     "composite": {
      "A": [
       59.165096184175106,
       74.53230995103505
      ],
      "B": [
       54.902945001197075,
       73.17537269212036
      ],
      "C": [
       41.08687235012159,
       74.53230995103505
      ]
     },
     "seed": 22
    },
    "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)",
    "ci95_n": 300,
    "complete": true,
    "equated": {
     "calibration": 85.15566582345811,
     "capability": 72.19088219725606,
     "intelligence": 59.226098571054024
    },
    "equating_flag": null,
    "equating_offset": {
     "C": 0.8237204845956931,
     "I": 2.574115857338972
    },
    "full_set_api": false,
    "g_med_used": 16.820628977585045,
    "lane": "api",
    "metric_basis": "A2 u P (sealed side = supplementary API subset A2, n=300)",
    "n_items": 600,
    "n_public": 300,
    "n_sealed_side": 300,
    "offset_ci95": {
     "C": [
      -2.9350424539001043,
      9.397802125135494
     ],
     "I": [
      -4.786775030874768,
      10.076945418608098
     ]
    },
    "on_S": null,
    "per_type": {
     "choice": {
      "accuracy": 0.7933333333333333,
      "calibration": 77.8355360065047,
      "cc": 73.01414581066375,
      "cc_by_type": {
       "choice": 73.01414581066375
      },
      "cc_tiered": 73.0476445854911,
      "cc_tiered_open": 82.78833309663467,
      "cc_tiered_sealed": 63.30695607434754,
      "low_n": false,
      "n": 300,
      "n_by_type": {
       "choice": 300
      },
      "score": 73.01414581066375
     },
     "noul": {
      "accuracy": 0.7266666666666667,
      "calibration": 87.09142854224856,
      "cc": 45.333333333333336,
      "cc_by_type": {
       "noul": 45.333333333333336
      },
      "cc_tiered": 39.285117056856194,
      "cc_tiered_open": 54.94202898550725,
      "cc_tiered_sealed": 23.62820512820513,
      "low_n": false,
      "n": 150,
      "n_by_type": {
       "noul": 150
      },
      "score": 45.333333333333336
     },
     "score": {
      "accuracy": 0.7466666666666667,
      "calibration": 87.1325461475575,
      "cc": 59.17272659737858,
      "cc_by_type": {
       "score": 59.17272659737858
      },
      "cc_tiered": 57.62318649879788,
      "cc_tiered_open": 63.288852399084945,
      "cc_tiered_sealed": 51.95752059851082,
      "low_n": false,
      "n": 150,
      "n_by_type": {
       "score": 150
      },
      "score": 59.17272659737858
     }
    },
    "raw_on_A": {
     "calibration": 84.01983689877025,
     "capability": 70.33590980624265,
     "intelligence": 56.65198271371505
    },
    "set": "A2+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 300
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 150
    },
    "score": {
     "invalid_rate": 0.00666666666666671,
     "n": 150
    }
   },
   "views": {
    "Balanced 33:33:33": 67.81219900588276,
    "Emphasis on Accuracy 60:20:20": 64.09539501387944,
    "Emphasis on Cost 20:20:60": 65.44947282925277,
    "Emphasis on Speed 20:60:20": 74.85522459350092,
    "Intelligence only": 59.22609857105403
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "nagisanzenin",
   "axes": {
    "calibration": 85.14254261258804,
    "cost": 29.98821307073638,
    "intelligence": 62.38818589248167,
    "speed": 85.82001330475426
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07628781316710302,
      "ece_score": 84.7424373665794,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.7424373665794,
      "score_before_time_limit": 84.7424373665794,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.1554959785522788,
      "brier": 0.10066812652875887,
      "ece": 0.08008409057957327,
      "n": 373,
      "score": 83.98318188408534,
      "score_before_time_limit": 83.98318188408534,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9327956989247311,
      "interval90_width": 0.258736559139785,
      "n": 372,
      "nrps": 0.07096566496914646,
      "score": 86.70200858709943,
      "score_before_time_limit": 86.70200858709943,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09749708164443252
     }
    },
    "score": 85.14254261258804
   },
   "capability": 73.76536425253485,
   "composite_ci95": {
    "A": [
     19.28216616131712,
     20.05230443196449
    ],
    "B": [
     19.529991464343766,
     20.72336298180548
    ],
    "C": [
     17.52753843215356,
     20.05230443196449
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.8-27B -> qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the evaluator's wrapper (input = attention-mask ones over every forward pass, output_tokens 0). OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.215638 per 1,000 decisions on the common basis (USD 0.213482 over all 1,500 items) is 3.338x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.215638 per 1,000 decisions (3.338x the official cost cap USD 0.064595), against USD 0.213482 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.21348208
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.21563846464646463
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nagi-ENORMOUS 27B (nagisanzenin, Qwen3.8-27B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's SDK loaded in-process (nagi.load_enormous) behind the evaluator's wrapper.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.77912378162935,
    "I_sealed": 56.997248003334,
    "base": 62.38818589248167,
    "excess": 0.9423490079396402,
    "gap": 10.781875778295351,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.75336686147108,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 75.46398238439761,
       "judge": 76.26895187870798,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 65.67632850241546,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 50.0,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 62.90767598100152,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.48231371851977,
       "hard": 64.9557142549451,
       "judge": 69.52499554842026,
       "standard": 48.59830121322711
      }
     },
     "sealed|choice": {
      "cc": 73.1916523589463,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 68.77529779114143,
       "judge": 74.43682664054849,
       "standard": 73.4233284117275
      }
     },
     "sealed|noul": {
      "cc": 47.06459263844751,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 68.96551724137932,
       "hard": 45.45454545454546,
       "judge": 38.46153846153846,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 50.7354990126082,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 68.6310910781143,
       "hard": 46.3612306455534,
       "judge": 49.43194058927436,
       "standard": 52.49157734896552
      }
     }
    }
   },
   "jevbench_score": 19.77217178721856,
   "key": "nagi-enormous",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nagisanzeninz/Nagi-ENORMOUS@0d8e052d698461ecd49c4f39a09f6bd0179a1d5c",
   "noul_decisive": {
    "acc_among_decisive": 0.9142857142857143,
    "decisive": 315,
    "decisive_rate": 0.8445040214477212,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.14254261258804,
     "cost": 29.98821307073638,
     "intelligence": 62.38818589248167,
     "speed": 86.16105911802293
    },
    "board_release": "v1.6.43",
    "capability": 73.76536425253485,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      19.294091538220005,
      20.0652017609536
     ],
     "B": [
      19.539777428272505,
      20.734381755643742
     ],
     "C": [
      17.53840599334894,
      20.0652017609536
     ]
    },
    "jevbench_score": 19.784711166742767,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 97,
    "rank": 132,
    "ranks": {
     "A": 132,
     "B": 126,
     "C": 116,
     "capability": 50
    },
    "scores": {
     "A": 19.784711166742767,
     "B": 20.264621768982312,
     "C": 19.784711166742767
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4519195217639208,
     "p50_s_raw": 0.1509597608819604,
     "p95_s_adjusted": 0.5355906491167843,
     "p95_s_raw": 0.19279532455839216
    },
    "views": {
     "Balanced 33:33:33": 17.696438535382505,
     "Emphasis on Accuracy 60:20:20": 19.331602164737944,
     "Emphasis on Cost 20:20:60": 14.087313813773893,
     "Emphasis on Speed 20:60:20": 21.36249462717993,
     "Intelligence only": 22.442101625156123
    }
   },
   "rank": 150,
   "ranked": true,
   "ranks": {
    "A": 150,
    "B": 145,
    "C": 133,
    "capability": 61
   },
   "repo": "https://huggingface.co/nagisanzeninz/Nagi-ENORMOUS",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 19.77217178721856,
    "B": 20.254096497135677,
    "C": 19.77217178721856
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4575406416319311,
    "p50_s_raw": 0.15377032081596553,
    "p95_s_adjusted": 0.5722278523724526,
    "p95_s_raw": 0.2111139261862263
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 74.14032183781576,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 47.4209655134473
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 74.14032183781576
      },
      "calibration_gen": {
       "accuracy": 0.6732673267326733,
       "cc": 54.21132684915775,
       "cc_by_type": {
        "choice": 68.18663838812301,
        "noul": 43.99999999999999,
        "score": 50.44734215935023
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 54.21132684915775
      },
      "composite_gen": {
       "accuracy": 0.8514851485148515,
       "cc": 79.6200178020504,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 52.77152585420901
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 79.6200178020504
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 82.42374252124812,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 87.27122756374432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 82.42374252124812
      },
      "jaggedness": {
       "accuracy": 0.5666666666666667,
       "cc": 25.273315341162046,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": -18.918918918918916,
        "score": 49.59930767580255
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 25.273315341162046
      },
      "llm_router": {
       "accuracy": 0.9105431309904153,
       "cc": 82.02065127885221,
       "cc_by_type": {
        "choice": 91.28290921898389,
        "noul": 95.8762886597938,
        "score": 58.90275595777894
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 82.02065127885221
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 6.669224859541649,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25.0,
        "score": 19.23447870233629
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 6.669224859541649
      },
      "multilingual": {
       "accuracy": 0.7345454545454545,
       "cc": 57.00736526474546,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 38.888888888888886,
        "score": 60.499688809488326
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.00736526474546
      },
      "negation_pair_gen": {
       "accuracy": 0.8488372093023255,
       "cc": 61.959669648822455,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 46.66666666666666,
        "score": 45.87900894646737
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 61.959669648822455
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 73.20537560651158,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 80.0,
        "score": 57.63414483755276
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 73.20537560651158
      },
      "usecase": {
       "accuracy": 0.7045454545454546,
       "cc": 54.04431686728915,
       "cc_by_type": {
        "choice": 68.1372549019608,
        "noul": 41.66666666666667,
        "score": 52.32902903323995
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.04431686728915
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 22.777334487427282,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -24.525139394861007
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 22.777334487427282
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 23.3844609455102,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 3.4867161698639237
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 23.3844609455102
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.78205425673583,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 75.239968079942
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.78205425673583
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 38.730441914975806,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 99.80398868719931
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 38.730441914975806
      },
      "en": {
       "accuracy": 0.7688966116420504,
       "cc": 62.08334185157287,
       "cc_by_type": {
        "choice": 74.95759143774178,
        "noul": 56.83453237410072,
        "score": 54.457901742876125
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 62.08334185157287
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.46236509889698,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 9.17502351855064
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.46236509889698
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.48213085184086,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 98.89994446809092
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.48213085184086
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 82.9428569359917,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 91.18151198444569
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 82.9428569359917
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 49.72408656861362,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25.0,
        "score": 57.691812778466556
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 49.72408656861362
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 58.09938627602248,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 2.357394993305184
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 58.09938627602248
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 21.74234134927224,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0.0,
        "score": -6.178853632882242
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 21.74234134927224
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -1.3052903918953547,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 40.528573268758386
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.0491525522278,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.1474576566834
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.0491525522278
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -20.38367168727892,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": -9.202963113784811
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 76.55637870233811,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 64.9040070338023
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 76.55637870233811
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 74.98400673194925,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 67.37983398488723
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.98400673194925
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 59.284793741948214,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 68.91821101307869
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 59.284793741948214
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 68.73685288229498,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": 49.68881951645017
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 68.73685288229498
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.03268404130108,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 90.35611664003227
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.03268404130108
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 54.74146896611071,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 30.891073564998806
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 54.74146896611071
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8,
       "cc": 74.1671291069481,
       "cc_by_type": {
        "choice": 74.1671291069481
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 74.1671291069481
      },
      "noul": {
       "accuracy": 0.768,
       "cc": 53.6,
       "cc_by_type": {
        "noul": 53.6
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 53.6
      },
      "score": {
       "accuracy": 0.6666666666666666,
       "cc": 54.012205085152345,
       "cc_by_type": {
        "score": 54.012205085152345
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 54.012205085152345
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -5.814728435567205,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -25.0,
        "score": -12.444185306701616
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 65.96125238263933,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 16.517297520588812
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 65.96125238263933
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 40.62716551888533,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100.0,
        "score": -61.858340841718
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 40.62716551888533
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 2.0832411267705617,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -33.51635849103334
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 2.0832411267705617
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 39.13973735127555,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 65.24529901034839
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 39.13973735127555
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -25.614597750300508,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -159.86266117542982
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.7333333333333333,
       "cc": 57.204794917237045,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 58.16900660045062
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.204794917237045
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 29.714186153960508,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50.0,
        "score": 60.048481806829244
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 29.714186153960508
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 0.8791936057861172,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 25.13758081735835
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.8791936057861172
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 55.57642841921321,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 66.66666666666667,
        "score": 51.45448154600507
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 55.57642841921321
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 51.56691874020171,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": 44.62323684076017
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 51.56691874020171
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": 11.911912271535977,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 31.434661545790732
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.911912271535977
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 72.50539113615088,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 69.03248818772326
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.50539113615088
      },
      "model_routing": {
       "accuracy": 0.8940092165898618,
       "cc": 78.06075539889343,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 93.93939393939394,
        "score": 46.087028101442215
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 78.06075539889343
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 17.71194333648423,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 64.65214478872332
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 17.71194333648423
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 35.85502771771261,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 74.32460033048231
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 35.85502771771261
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 30.833875246891477,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": 22.058275986979837
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.833875246891477
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 21.590370642594802,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 56.07545975387135
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 21.590370642594802
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.11485034921368,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.34455104764109
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.11485034921368
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.73695118367363,
      87.37702121824532
     ],
     "I": [
      57.11710491178149,
      66.34463835228868
     ],
     "capability": [
      69.6246461730435,
      76.14865327125807
     ],
     "composite": {
      "A": [
       19.28216616131712,
       20.05230443196449
      ],
      "B": [
       19.529991464343766,
       20.72336298180548
      ],
      "C": [
       17.52753843215356,
       20.05230443196449
      ]
     },
     "seed": 50
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.14254261258804,
     "capability": 73.76536425253485,
     "intelligence": 62.38818589248167
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8,
      "calibration": 84.7424373665794,
      "cc": 74.1671291069481,
      "cc_by_type": {
       "choice": 74.1671291069481
      },
      "cc_tiered": 73.97250961020869,
      "cc_tiered_open": 74.75336686147108,
      "cc_tiered_sealed": 73.1916523589463,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.1671291069481
     },
     "noul": {
      "accuracy": 0.768,
      "calibration": 83.98318188408534,
      "cc": 53.6,
      "cc_by_type": {
       "noul": 53.6
      },
      "cc_tiered": 56.37046057043148,
      "cc_tiered_open": 65.67632850241546,
      "cc_tiered_sealed": 47.06459263844751,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.6
     },
     "score": {
      "accuracy": 0.6666666666666666,
      "calibration": 86.70200858709943,
      "cc": 54.012205085152345,
      "cc_by_type": {
       "score": 54.012205085152345
      },
      "cc_tiered": 56.82158749680486,
      "cc_tiered_open": 62.90767598100152,
      "cc_tiered_sealed": 50.7354990126082,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.012205085152345
     }
    },
    "raw_on_A": {
     "calibration": 84.73824539717386,
     "capability": 73.41666063559657,
     "intelligence": 62.09507587401927
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 17.683064142138775,
    "Emphasis on Accuracy 60:20:20": 19.32202355672641,
    "Emphasis on Cost 20:20:60": 14.08222658273634,
    "Emphasis on Speed 20:60:20": 21.327444146428867,
    "Intelligence only": 22.442101625156123
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "nagisanzenin",
   "axes": {
    "calibration": 73.2732669877037,
    "cost": null,
    "intelligence": 54.80664491424396,
    "speed": 88.38293814019335
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14730316908633123,
      "ece_score": 70.53936618273376,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 70.53936618273376,
      "score_before_time_limit": 70.53936618273376,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.1420911528150134,
      "brier": 0.13578925025367564,
      "ece": 0.11265263438596824,
      "n": 373,
      "score": 77.46947312280635,
      "score_before_time_limit": 77.46947312280635,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8521505376344086,
      "interval90_width": 0.2540322580645162,
      "n": 372,
      "nrps": 0.1041654308585557,
      "score": 71.81096165757103,
      "score_before_time_limit": 71.81096165757103,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.22980766799501195
     }
    },
    "score": 73.2732669877037
   },
   "capability": 64.03995595097383,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base google/gemma-4-12B-it (the module lists it in UNLISTED_BASE_MODELS, so reference() returns no price); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the evaluator's wrapper (attention-mask ones over every forward pass: input 764,280, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nagi-HUGE 12B (nagisanzenin, Gemma-4-12B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's SDK loaded in-process (nagi.load_huge) behind the evaluator's wrapper.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 59.531691789779515,
    "I_sealed": 50.081598038708414,
    "base": 54.80664491424396,
    "excess": -0.3894330192846098,
    "gap": 9.450093751071101,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 71.98506861847196,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 68.35860250494397,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 52.719806763285035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 53.89019998758157,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.56725028103613,
       "hard": 59.87843833211595,
       "judge": 57.63522131418832,
       "standard": 33.4576661618754
      }
     },
     "sealed|choice": {
      "cc": 68.46211926245671,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 62.530357349369716,
       "judge": 68.26640548481882,
       "standard": 72.47416156928918
      }
     },
     "sealed|noul": {
      "cc": 44.28703360458379,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 41.81818181818182,
       "judge": 29.230769230769237,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 37.49564124908475,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 56.18011440890034,
       "hard": 32.5186886432668,
       "judge": 31.435392540978093,
       "standard": 47.19768294297284
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "nagi-huge",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nagisanzeninz/Nagi-HUGE@3e0a58b19116afca7abf7f854d82ba85f5de5c4b",
   "not_ranked_because": "unpriced: google/gemma-4-12B-it is in the frozen module's UNLISTED_BASE_MODELS (no eligible base-model price); one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.878125,
    "decisive": 320,
    "decisive_rate": 0.8579088471849866,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.2732669877037,
     "cost": null,
     "intelligence": 54.80664491424396,
     "speed": 88.62940634950104
    },
    "board_release": "v1.6.43",
    "capability": 64.03995595097383,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.34666025787591936,
     "p50_s_raw": 0.09833012893795967,
     "p95_s_adjusted": 0.3955080335494131,
     "p95_s_raw": 0.12275401677470654
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/nagisanzeninz/Nagi-HUGE",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.35058780908584597,
    "p50_s_raw": 0.10029390454292297,
    "p95_s_adjusted": 0.4139132943470031,
    "p95_s_raw": 0.13195664717350153
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 80.8965045441356,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 59.35618029907346
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 80.8965045441356
      },
      "calibration_gen": {
       "accuracy": 0.5742574257425742,
       "cc": 41.022478364333296,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": 43.99999999999999,
        "score": 36.331485994378454
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 41.022478364333296
      },
      "composite_gen": {
       "accuracy": 0.7227722772277227,
       "cc": 54.3475547262935,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 40.74074074074075,
        "score": 48.318183600741364
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 54.3475547262935
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 88.61842154243884,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 65.8552646273165
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 88.61842154243884
      },
      "jaggedness": {
       "accuracy": 0.5666666666666667,
       "cc": 27.382018300134657,
       "cc_by_type": {
        "choice": 52.358036573628496,
        "noul": 2.7027027027026973,
        "score": 27.08531562407278
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 27.382018300134657
      },
      "llm_router": {
       "accuracy": 0.8466453674121406,
       "cc": 74.9947462424799,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 83.50515463917525,
        "score": 59.70572844857087
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 74.9947462424799
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 20.044670000518725,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 0.0,
        "score": -15.123721957206726
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 20.044670000518725
      },
      "multilingual": {
       "accuracy": 0.730909090909091,
       "cc": 57.472837311308034,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": 52.77777777777777,
        "score": 49.96352515712449
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.472837311308034
      },
      "negation_pair_gen": {
       "accuracy": 0.6744186046511628,
       "cc": 31.85947400478956,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": -6.666666666666665,
        "score": 25.578422014368694
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 31.85947400478956
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 81.2963361578178,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100.0,
        "score": 70.91603550048045
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.2963361578178
      },
      "usecase": {
       "accuracy": 0.6363636363636364,
       "cc": 48.42747769597158,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": 41.66666666666667,
        "score": 40.380472303601
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 48.42747769597158
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 7.208501158443632,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": -90.27925842943101
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 7.208501158443632
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 52.975324368289364,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 46.32996696047484
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.975324368289364
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.10830849392268,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100.0,
        "score": 84.18333256141403
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.10830849392268
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.5874788778112,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.94865637272596
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.5874788778112
      },
      "en": {
       "accuracy": 0.7072111207645526,
       "cc": 52.781473595451,
       "cc_by_type": {
        "choice": 69.0264946729964,
        "noul": 47.482014388489205,
        "score": 41.835911724867394
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 52.781473595451
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.1400106830659,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 29.2079602710574
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.1400106830659
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.7975629613487,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.84624079661444
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.7975629613487
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 84.29596567394786,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 74.06436761007889
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.29596567394786
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 41.946148025454015,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 25.0,
        "score": 0.8384440763620549
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 41.946148025454015
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 71.63085300784105,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 28.922177106142033
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.63085300784105
      },
      "ja": {
       "accuracy": 0.44,
       "cc": 12.01915193855757,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 0.0,
        "score": 7.542761613925297
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 12.01915193855757
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 64.63002166245748,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0,
        "score": 38.33450943181689
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 64.63002166245748
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.16468569071083,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.4940570721325
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.16468569071083
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 29.144393403202685,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 6.047898824326681
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 29.144393403202685
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 56.66691195109527,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 19.999999999999996,
        "score": 57.61817131667982
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.66691195109527
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.41158446563735,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 53.6625671859515
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.41158446563735
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 44.08012594525695,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 23.304207623004913
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 44.08012594525695
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 38.30087713735259,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50.0,
        "score": -13.358238153159618
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 38.30087713735259
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 81.29992212215188,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 76.15783088258468
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 81.29992212215188
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 37.21820113197382,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -21.678729937411866
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 37.21820113197382
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7653333333333333,
       "cc": 69.68943148548576,
       "cc_by_type": {
        "choice": 69.68943148548576
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.68943148548576
      },
      "noul": {
       "accuracy": 0.7493333333333333,
       "cc": 49.86666666666666,
       "cc_by_type": {
        "noul": 49.86666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 49.86666666666666
      },
      "score": {
       "accuracy": 0.584,
       "cc": 42.51279672323708,
       "cc_by_type": {
        "score": 42.51279672323708
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 42.51279672323708
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 18.68109795677604,
       "cc_by_type": {
        "choice": 52.0,
        "noul": 0.0,
        "score": 4.043293870328113
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 18.68109795677604
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 45.116373885865976,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 77.77777777777777,
        "score": -5.161575374838234
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 45.116373885865976
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 50.81996606870461,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 34.98022340936585
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 50.81996606870461
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -18.91273698918377,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": -29.83762617222967
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 50.965624322682,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 48.54904688108945
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 50.965624322682
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -45.01234945271016,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 0.0,
        "score": -184.72258294932544
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 34.253607619886445,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 27.690794848454857
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.253607619886445
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 4.190740847971121,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -100.0,
        "score": 54.38406923380883
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 4.190740847971121
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 50.32081411651301,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 53.46244234953903
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 50.32081411651301
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 68.33470459909553,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100.0,
        "score": 43.548011013560675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 68.33470459909553
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 49.021948938186966,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 3.1381982357495453
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 49.021948938186966
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 31.490484817207232,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 25.65425015054643
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 31.490484817207232
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 68.93444315573477,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 46.80332946720428
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.93444315573477
      },
      "model_routing": {
       "accuracy": 0.8663594470046083,
       "cc": 78.12484590988917,
       "cc_by_type": {
        "choice": 87.14285714285714,
        "noul": 100.0,
        "score": 47.2316805868104
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 78.12484590988917
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 39.09003196881363,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 40.302725464982146
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 39.09003196881363
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 87.47734221450266,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": 79.14511856551354
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 87.47734221450266
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 33.94641567016526,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 16.617572133648473
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 33.94641567016526
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 28.08332853716604,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 33.33333333333333,
        "score": 37.87317401729523
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 28.08332853716604
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 95.87265733076127,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 98.2941285759137
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 95.87265733076127
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.82121675320265,
      76.3898801463225
     ],
     "I": [
      49.533417058444485,
      59.0670299888095
     ],
     "capability": [
      59.842759105481505,
      67.1284484360694
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.2732669877037,
     "capability": 64.03995595097383,
     "intelligence": 54.80664491424396
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 70.53936618273376,
      "cc": 69.68943148548576,
      "cc_by_type": {
       "choice": 69.68943148548576
      },
      "cc_tiered": 70.22359394046433,
      "cc_tiered_open": 71.98506861847196,
      "cc_tiered_sealed": 68.46211926245671,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.68943148548576
     },
     "noul": {
      "accuracy": 0.7493333333333333,
      "calibration": 77.46947312280635,
      "cc": 49.86666666666666,
      "cc_by_type": {
       "noul": 49.86666666666666
      },
      "cc_tiered": 48.50342018393441,
      "cc_tiered_open": 52.719806763285035,
      "cc_tiered_sealed": 44.28703360458379,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 49.86666666666666
     },
     "score": {
      "accuracy": 0.584,
      "calibration": 71.81096165757103,
      "cc": 42.51279672323708,
      "cc_by_type": {
       "score": 42.51279672323708
      },
      "cc_tiered": 45.69292061833316,
      "cc_tiered_open": 53.89019998758157,
      "cc_tiered_sealed": 37.49564124908475,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.51279672323708
     }
    },
    "raw_on_A": {
     "calibration": 73.92409836881791,
     "capability": 64.51989601789064,
     "intelligence": 55.11569366696338
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pst2154",
   "axes": {
    "calibration": 89.03920345557668,
    "cost": null,
    "intelligence": 49.60038985253669,
    "speed": 87.35910345792061
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.031840143420803706,
      "ece_score": 93.63197131583925,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 93.63197131583925,
      "score_before_time_limit": 93.63197131583925,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.30400000000000005,
      "brier": 0.10770403214312142,
      "ece": 0.06945449416016128,
      "n": 375,
      "score": 86.10910116796775,
      "score_before_time_limit": 86.10910116796775,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.928,
      "interval90_width": 0.37866666666666676,
      "n": 375,
      "nrps": 0.09148110518890483,
      "score": 87.37653788292307,
      "score_before_time_limit": 87.37653788292307,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0804940685763168
     }
    },
    "score": 89.03920345557668
   },
   "capability": 69.31979665405669,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module RAISES on the declared base nvidia/Nemotron-Labs-Diffusion-8B ('base model lacks an explicit market-listing decision') and the dated snapshot carries no listing of that family, so this row carries no pricing_base and is listed, measured and shown WITHOUT a cost number and WITHOUT a rank - the pre-declared default (Sage 1.4, Celeris Option A). No rate is invented. The predecessor nemotron-diffusion-8b row's usd_per_1000_override is a v1.5.4 historical carry on a different draw and is deliberately NOT reused. Asked of the deputy on board #19427; the server reports both token fields, so a later pricing decision needs no re-measurement.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nemotron Deciffusion 8B v2 (pst2154)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (United States; measurement receipts on file), 1x NVIDIA RTX A6000 48 GB - different hardware from the L40S and GB200 the author quotes and from the H100/RTX 5090 rows of this draw. Started from the author's OWN published container, by DIGEST ghcr.io/pst2154/nemotron-deciffusion@sha256:944c0af4967ebd1e461a3b1faa9d16db197288af24393b21c917617461f35276 (image config sha256:0a4bd2d039b5c2da9ff8b181232ee3714f6b38e31bd9a3100a29326068d6f802), in the author's own documented sandboxed form: docker run --gpus all --shm-size=8g --network none --read-only --tmpfs /tmp:rw,exec,size=16g --user 1000:1000 --cap-drop ALL --security-opt no-new-privileges -v <checkpoint>:/models/checkpoint:ro, with QUANTIZATION=none (BF16) and MAX_MODEL_LEN=16384 and every other default of the image (ATTENTION=noncausal, COMPACT_OVER=8192, COMPACT_BUDGET=8192, MAX_BATCHED_TOKENS=65536, --no-enable-prefix-caching, --max-logprobs 1052, --logprobs-mode processed_logits, vLLM nightly fe4df3df0eb7 with the author's overlay). /health reported back quantization none, attention noncausal, max_prompt_tokens 16382. All ten checkpoint files verified against the author's own SHA256SUMS (sha256sum -c, 10/10 OK) BEFORE the container started. One request at a time from the unchanged runner, which reached the server through the container's own network namespace with nsenter - the container published no port and had no network at all. The cut was proven before the sealed input was staged (only lo, no routes, a name probe and a bare-IP probe both failed) and re-proven immediately before the first sealed request, with a synthetic smoke gate passed in between (custody #19428). 1,500/1,500 answered, HTTP 200 on every item, raw p50 105 ms / p95 180 ms / max 1.82 s.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 55.54041453254173,
    "I_sealed": 43.660365172531655,
    "base": 49.60038985253669,
    "excess": 2.0405225896543655,
    "gap": 11.880049360010076,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.4454905515748,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 84.90091223655237,
       "judge": 72.31377719182598,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 44.98550724637681,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 39.13043478260869,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 46.19024579967359,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.27224719217729,
       "hard": 41.00474552808914,
       "judge": 52.73987117942826,
       "standard": 34.19580757695866
      }
     },
     "sealed|choice": {
      "cc": 66.84585672814276,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 65.65282757025557,
       "judge": 56.80705190989228,
       "standard": 71.52499472685088
      }
     },
     "sealed|noul": {
      "cc": 23.96819855389078,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 44.827586206896555,
       "hard": 27.27272727272727,
       "judge": 7.692307692307687,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 40.167040235561444,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 62.908566475040885,
       "hard": 39.62773652825137,
       "judge": 34.70949534264651,
       "standard": 38.061201869814276
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "nemotron-deciffusion-8b-v2",
   "last_measured_on": "2026-10-15",
   "licence": "NVIDIA Nemotron Open Model License (https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/); base nvidia/Nemotron-Labs-Diffusion-8B",
   "listing": "listed",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nemotron-deciffusion/Nemotron-Deciffusion-8B-v2@1dea38015e86e101eb978a33fa493818fc71e366",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base nvidia/Nemotron-Labs-Diffusion-8B and raises rather than estimating; the deputy's answer to #19427 decides whether an entry is added",
   "noul_decisive": {
    "acc_among_decisive": 0.9501915708812261,
    "decisive": 261,
    "decisive_rate": 0.696,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.03920345557668,
     "cost": null,
     "intelligence": 49.60038985253669,
     "speed": 87.70433528726323
    },
    "board_release": "v1.6.43",
    "capability": 69.31979665405669,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.35293248631060126,
     "p50_s_raw": 0.10146624315530062,
     "p95_s_adjusted": 0.48070078976452346,
     "p95_s_raw": 0.16535039488226172
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/nemotron-deciffusion/Nemotron-Deciffusion-8B-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3605954047292471,
    "p50_s_raw": 0.10529770236462355,
    "p95_s_adjusted": 0.5094123460352421,
    "p95_s_raw": 0.17970617301762104
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 69.81508925384576,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 17.778601094870574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 69.81508925384576
      },
      "calibration_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 24.130548030105405,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": -19.999999999999996,
        "score": 36.930350346943996
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 24.130548030105405
      },
      "composite_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 62.586444350419164,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 40.74074074074075,
        "score": 56.77468987149234
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 62.586444350419164
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 76.48515065368318,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 69.45545196104952
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 76.48515065368318
      },
      "jaggedness": {
       "accuracy": 0.5611111111111111,
       "cc": 22.292630536155855,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": -18.918918918918916,
        "score": 21.889207062516412
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 22.292630536155855
      },
      "llm_router": {
       "accuracy": 0.7635782747603834,
       "cc": 64.81045782636264,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 79.38144329896907,
        "score": 49.91829330418326
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 64.81045782636264
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": -14.49955948045043,
       "cc_by_type": {
        "choice": 48.387096774193544,
        "noul": -100.0,
        "score": 8.114224784455171
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -14.464526914190303,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -25.0,
        "score": -31.795642598241013
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 43.55075461566889,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 16.666666666666675,
        "score": 48.22100637499275
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 43.55075461566889
      },
      "negation_pair_gen": {
       "accuracy": 0.7209302325581395,
       "cc": 40.54640345029721,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": 6.666666666666665,
        "score": 28.305877017558302
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 40.54640345029721
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 76.80928525816542,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100.0,
        "score": 57.45488280152327
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.80928525816542
      },
      "usecase": {
       "accuracy": 0.6772727272727272,
       "cc": 46.174276523358934,
       "cc_by_type": {
        "choice": 70.58823529411767,
        "noul": 25.0,
        "score": 42.934594275959135
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 46.174276523358934
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 7.324936282948684,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 100.0,
        "score": -101.83471496067776
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 7.324936282948684
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 43.555049861840054,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 18.06914344112691
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 43.555049861840054
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 60.20540912770944,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75.0,
        "score": 37.474634462774326
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 60.20540912770944
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 32.087424203928286,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 96.44849235107722
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 32.087424203928286
      },
      "en": {
       "accuracy": 0.7046046915725456,
       "cc": 49.66247129995006,
       "cc_by_type": {
        "choice": 72.10187818064216,
        "noul": 33.81294964028776,
        "score": 43.07258607892028
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 49.66247129995006
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 67.89447005427888,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75.0,
        "score": 38.47133838469635
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.89447005427888
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 46.07913376963507,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 70.47783846737516
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 46.07913376963507
      },
      "fr": {
       "accuracy": 0.42857142857142855,
       "cc": 23.839275340930822,
       "cc_by_type": {
        "choice": 15.294117647058824,
        "noul": 0.0,
        "score": 56.223708375733636
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 23.839275340930822
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 27.037410682068597,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 0.0,
        "score": 31.39156165514434
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 27.037410682068597
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 67.26700899902467,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 15.830645079692863
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.26700899902467
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 22.223201710577673,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 3.857849770335098
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 22.223201710577673
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 49.120207596142166,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 47.36062278842651
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 49.120207596142166
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.3181021434982,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 95.62097309716124
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.3181021434982
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 0.6113352439164336,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 27.80803170577527
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.6113352439164336
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 53.9772539570996,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 57.16663279808676
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 53.9772539570996
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 67.38198878819895,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 55.18073360089622
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 67.38198878819895
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 26.13951124323292,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 35.01427841054983
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 26.13951124323292
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 29.199516858326863,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 31.076811444545815
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.199516858326863
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.32623665099804,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 81.4948389852522
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.32623665099804
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 31.384878910972457,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -13.754967786856632
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.384878910972457
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7693333333333333,
       "cc": 70.2060889033468,
       "cc_by_type": {
        "choice": 70.2060889033468
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 70.2060889033468
      },
      "noul": {
       "accuracy": 0.6613333333333333,
       "cc": 32.266666666666666,
       "cc_by_type": {
        "noul": 32.266666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 32.266666666666666
      },
      "score": {
       "accuracy": 0.5893333333333334,
       "cc": 42.502298232284154,
       "cc_by_type": {
        "score": 42.502298232284154
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 42.502298232284154
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 19.398592732098425,
       "cc_by_type": {
        "choice": 36.0,
        "noul": 0.0,
        "score": 22.19577819629528
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 19.398592732098425
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 4.560003558944312,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -22.896938943594947
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.560003558944312
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 54.984605803255306,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 63.73430521464396
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 54.984605803255306
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.94550967102398,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -28.66109041658069
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 22.84201437357555,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 45.337637323625216
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.84201437357555
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": 12.742920575752285,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 33.33333333333333,
        "score": -19.63287349286893
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 12.742920575752285
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 36.722035568569,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 25.29215712587507
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 36.722035568569
      },
      "gaming": {
       "accuracy": 0.5454545454545454,
       "cc": 28.13714675856338,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": 0.0,
        "score": 36.67624863805947
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 28.13714675856338
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 18.23892289455222,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60.0,
        "score": 56.38343535032333
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.23892289455222
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 43.4296215704276,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 33.33333333333333,
        "score": 35.49942859422358
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 43.4296215704276
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 50.10373799055933,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 28.60578761508887
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 50.10373799055933
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 15.263876228821458,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -33.333333333333336,
        "score": 11.383026535926744
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 15.263876228821458
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 78.76271298376912,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 100.0,
        "score": 47.804453730577976
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 78.76271298376912
      },
      "model_routing": {
       "accuracy": 0.7880184331797235,
       "cc": 67.78953178167247,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 93.93939393939394,
        "score": 37.48114945757148
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.78953178167247
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 11.731104036301266,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 69.74225644671569
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 11.731104036301266
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 85.90646185176752,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 57.719385555302516
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 85.90646185176752
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 7.194103790460911,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": 30.695611863993566
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 7.194103790460911
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": -11.791006340242546,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100.0,
        "score": 22.59799547202598
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 79.5102568122854,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 88.53077043685622
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 79.5102568122854
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.13996631431465,
      89.24571617175492
     ],
     "I": [
      44.496386091996314,
      53.73836773548576
     ],
     "capability": [
      64.62383573654232,
      70.61089490969697
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 52
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.03920345557668,
     "capability": 69.31979665405669,
     "intelligence": 49.60038985253669
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7693333333333333,
      "calibration": 93.63197131583925,
      "cc": 70.2060889033468,
      "cc_by_type": {
       "choice": 70.2060889033468
      },
      "cc_tiered": 71.14567363985879,
      "cc_tiered_open": 75.4454905515748,
      "cc_tiered_sealed": 66.84585672814276,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.2060889033468
     },
     "noul": {
      "accuracy": 0.6613333333333333,
      "calibration": 86.10910116796775,
      "cc": 32.266666666666666,
      "cc_by_type": {
       "noul": 32.266666666666666
      },
      "cc_tiered": 34.47685290013379,
      "cc_tiered_open": 44.98550724637681,
      "cc_tiered_sealed": 23.96819855389078,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.266666666666666
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 87.37653788292307,
      "cc": 42.502298232284154,
      "cc_by_type": {
       "score": 42.502298232284154
      },
      "cc_tiered": 43.17864301761752,
      "cc_tiered_open": 46.19024579967359,
      "cc_tiered_sealed": 40.167040235561444,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.502298232284154
     }
    },
    "raw_on_A": {
     "calibration": 84.05380304593929,
     "capability": 67.14117186884195,
     "intelligence": 50.228540691744605
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "EryriLabs",
   "axes": {
    "calibration": 79.60239191738147,
    "cost": 71.03778132505249,
    "intelligence": 20.23194846537782,
    "speed": 54.82852093556197
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12475073319463456,
      "ece_score": 75.04985336107308,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 75.04985336107308
     },
     "noul": {
      "abstention_rate": 0.4879356568364611,
      "brier": 0.19795608560606281,
      "ece": 0.11196353518746716,
      "n": 373,
      "score": 77.60729296250656
     },
     "score": {
      "interval90_coverage": 0.9301075268817204,
      "interval90_width": 0.6832437275985659,
      "n": 372,
      "nrps": 0.1405294657671992,
      "score": 86.15002942856476,
      "top_ece": 0.06823497283075278
     }
    },
    "score": 79.60239191738147
   },
   "capability": 49.91717019137964,
   "composite_ci95": {
    "A": [
     3.909429912465545,
     10.019247475616854
    ],
    "B": [
     3.09621130589029,
     8.449477612845056
    ],
    "C": [
     2.714881883656628,
     6.957810746956148
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 dcfcecb0..., the module in force; its base-model references are unchanged since the dae86740... module this row was first priced with - the two later re-freezes each added one hosted-API launch-list key and nothing else): declared base Qwen/Qwen3.5-2B-Base -> deepinfra:Qwen/Qwen3.5-2B USD 0.02/M input, USD 0.1/M output. The author's shim reports usage {calls, permutations, backend} and NO token fields at all, so the input tokens are counted OFFLINE with the author's own prompt code (nev.layout.single_prompts over hev.decider_layout.render_row) at the exact serving settings, and cross-checked against llama-server's own prompt-token counter delta from the measured pass; output tokens are 0, because one readout position per call is restricted to the option letters and nothing is returned as text (the pjev-2b convention and the 9 Oct CoS rule for native option-probability models). Estimated, not charged, and not a quote from the author. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.009234 per 1,000 decisions (0.143x the official cost cap USD 0.064595), against USD 0.009142 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009142026666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00923437037037037
   },
   "decision_limit": null,
   "display": "Nev-2B (EryriLabs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own SHARED CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2, no AVX-512), NOT a dedicated machine and NOT a GPU: other jobs ran beside it throughout and the 1-minute load average was sampled every 60 s (measurement receipts on file) - over this row's window of 199 samples it ran 6.89 to 35.91, mean 13.06. The author's own two documented processes, unedited: llama-server -m nev-Q8_0.gguf -c 8192 --port 8080 --metrics (llama.cpp b11372, the one build the author names; --metrics is the only addition and changes no model behaviour), and the author's shim PYTHONPATH=python python -m nev.serve --backend llama-server:http://127.0.0.1:8080 --gguf nev-Q8_0.gguf on port 8090, with --backend-arg timeout=900 (the author's own documented flag, raised from the 300 s default because a loaded shared box can otherwise turn an answerable item into a false refusal). llama.cpp chose n_threads=6, its own default; the author passes none. One request at a time from the unchanged runner. Isolation: a user+network namespace (unshare -Urn, loopback only - Sandy permits no privileged unshare -n), and the author's two processes additionally inside bwrap with a cleared environment, a read-only root, tmpfs /home and /tmp and read-only binds of only the GGUF, the llama.cpp build, the venv, its interpreter and the author's own python/ tree, so the sealed item directory does not exist inside that sandbox; the runner stays outside it and the author's code never sees the file. All three cuts were PROVEN before the sealed input was read (only lo, no routes, a name probe and a bare-IP probe both failing, and an ls of the sealed tree failing inside the sandbox) and the network cut was re-proven immediately before the first sealed request, with a synthetic smoke gate passed in between (custody #19428). 1,485/1,500 answered, the 15 refusals all HTTP 502 relaying llama-server's own 400 exceed_context_size_error. Raw p50 7.02 s / p95 15.29 s / max 56.88 s - this is the shared box, not the model, and it is not comparable to the GPU and API rows of this draw.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 23.36677716577721,
    "I_sealed": 17.09711976497843,
    "base": 20.23194846537782,
    "excess": -3.5698693695569297,
    "gap": 6.269657400798781,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 44.315480566561476,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 64.97297297297297,
       "hard": 43.37842088707141,
       "judge": 36.71720500988794,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -4.347826086956519,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 25.784850930770162,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 55.23566077335427,
       "hard": 18.910462877818656,
       "judge": 28.909830224325873,
       "standard": 20.120753175047547
      }
     },
     "sealed|choice": {
      "cc": 42.28746532429039,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 61.88055908513342,
       "hard": 31.30565514051115,
       "judge": 47.992164544564176,
       "standard": 45.89748998101666
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -13.793103448275868,
       "hard": -14.54545454545455,
       "judge": -32.30769230769231,
       "standard": -4.477611940298509
      }
     },
     "sealed|score": {
      "cc": 9.003893970644905,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 8.190773300943444,
       "hard": 5.502957884946635,
       "judge": 6.841648353006702,
       "standard": 19.655694903349474
      }
     }
    }
   },
   "jevbench_score": 6.944821492706128,
   "key": "nev-2b",
   "last_measured_on": "2026-10-15",
   "licence": "Apache-2.0 (weights and shim); base Qwen/Qwen3.5-2B-Base Apache-2.0",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.6",
   "model_pin": "EryriLabs/Nev-2B-GGUF@65d329f676d75cc538346e2687bed71187ae76b2",
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": 0.8324607329842932,
    "decisive": 191,
    "decisive_rate": 0.5120643431635389,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.60239191738147,
     "cost": 71.03778132505249,
     "intelligence": 20.23194846537782,
     "speed": 54.82852093556197
    },
    "board_release": "v1.6.43",
    "capability": 49.91717019137964,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      3.909429912465545,
      10.019247475616854
     ],
     "B": [
      3.09621130589029,
      8.449477612845056
     ],
     "C": [
      2.714881883656628,
      6.957810746956148
     ]
    },
    "jevbench_score": 6.944821492706128,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": 141,
    "rank": 181,
    "ranks": {
     "A": 181,
     "B": 183,
     "C": 182,
     "capability": 174
    },
    "scores": {
     "A": 6.944821492706128,
     "B": 5.695775807883039,
     "C": 4.822792703268145
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 12.836194814974442,
     "p50_s_raw": 6.343097407487221,
     "p95_s_adjusted": 25.627816557881303,
     "p95_s_raw": 12.738908278940652
    },
    "views": {
     "Balanced 33:33:33": 6.009092360003425,
     "Emphasis on Accuracy 60:20:20": 4.533120029666712,
     "Emphasis on Cost 20:20:60": 7.449403907098574,
     "Emphasis on Speed 20:60:20": 6.924924518443358,
     "Intelligence only": 3.3126314570949202
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/EryriLabs/Nev-2B-GGUF",
   "scores": {
    "A": 6.944821492706128,
    "B": 5.695775807883039,
    "C": 4.822792703268145
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 12.836194814974442,
    "p50_s_raw": 6.343097407487221,
    "p95_s_adjusted": 25.627816557881303,
    "p95_s_raw": 12.738908278940652
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 46.938355311096586,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": -25.851600733376934
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 46.938355311096586
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 6.186350760967346,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -12,
        "score": -2.6328883321562913
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 6.186350760967346
      },
      "composite_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 0.4670486607000252,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -33.333333333333336,
        "score": -0.2248702780625278
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.4670486607000252
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 29.939866445263462,
       "cc_by_type": {
        "choice": 100,
        "noul": -19.999999999999996,
        "score": 9.819599335790386
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 29.939866445263462
      },
      "jaggedness": {
       "accuracy": 0.29444444444444445,
       "cc": -15.553356623470975,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -62.16216216216216,
        "score": 0.6801481146558652
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5814696485623003,
       "cc": 35.257392136280366,
       "cc_by_type": {
        "choice": 48.48991811217751,
        "noul": 21.649484536082465,
        "score": 35.632773760581124
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 35.257392136280366
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -10.080419559021554,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -50,
        "score": -6.014454553353321
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5854545454545454,
       "cc": 27.40677901913386,
       "cc_by_type": {
        "choice": 57.93935441799805,
        "noul": -2.777777777777779,
        "score": 27.058760417181315
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 27.40677901913386
      },
      "negation_pair_gen": {
       "accuracy": 0.29069767441860467,
       "cc": -24.37540474381098,
       "cc_by_type": {
        "choice": 23.333333333333332,
        "noul": -86.66666666666667,
        "score": -9.792880898099598
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.5208333333333334,
       "cc": 38.748335552596544,
       "cc_by_type": {
        "choice": 38.748335552596544
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 38.748335552596544
      },
      "perturb_pair": {
       "accuracy": 0.578125,
       "cc": 29.40536694686776,
       "cc_by_type": {
        "choice": 45.94594594594596,
        "noul": 19.999999999999996,
        "score": 22.270154894657324
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 29.40536694686776
      },
      "usecase": {
       "accuracy": 0.4909090909090909,
       "cc": 15.280162860434373,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": -25,
        "score": 17.4091160322835
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 15.280162860434373
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -1.277013365806873,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -27.640563906944426
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 48.03437833196489,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 31.50712885150143
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 48.03437833196489
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 64.63827546988519,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 42.8086317193901
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 64.63827546988519
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 12.633202749033721,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 88.36515759533208
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 12.633202749033721
      },
      "en": {
       "accuracy": 0.45438748913987836,
       "cc": 8.999756883752474,
       "cc_by_type": {
        "choice": 38.93167034817731,
        "noul": -21.58273381294964,
        "score": 9.650334116029747
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 8.999756883752474
      },
      "es": {
       "accuracy": 0.6774193548387096,
       "cc": 41.361890062571156,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 50,
        "score": -6.338473368567121
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 41.361890062571156
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -12.72873038858315,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": -6.492202094711197
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": 31.924464844536768,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 59.30280629831618
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 31.924464844536768
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 10.330713957506195,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": -10.488305054855719
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 10.330713957506195
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 68.38283399191003,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 19.178120058348945
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.38283399191003
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 29.951924796051298,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 27.04401902675596
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.951924796051298
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 26.885351406385215,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 25.100498663600103
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 26.885351406385215
      },
      "mixed": {
       "accuracy": 0.7142857142857143,
       "cc": 61.38675762936513,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 33.33333333333333,
        "score": 84.1602728880954
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 61.38675762936513
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 5.740053731306311,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 2.501546475304217
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.740053731306311
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 35.205060203589944,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0,
        "score": 20.850051537557825
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 35.205060203589944
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 8.889197461537425,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": -19.999999999999996,
        "score": 20.91626651543136
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 8.889197461537425
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -10.185723298219905,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 6.038574786191342
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -19.83194292150323,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -50,
        "score": -22.53930702537925
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.98909963559849,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 60.967298906795484
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.98909963559849
      },
      "zh": {
       "accuracy": 0.6,
       "cc": -3.638185596880632,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -52.15749464374924
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.56,
       "cc": 43.1676840352858,
       "cc_by_type": {
        "choice": 43.1676840352858
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 43.1676840352858
      },
      "noul": {
       "accuracy": 0.424,
       "cc": -15.200000000000003,
       "cc_by_type": {
        "noul": -15.200000000000003
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.4,
       "cc": 13.212269569733326,
       "cc_by_type": {
        "score": 13.212269569733326
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 13.212269569733326
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": 6.118021937826218,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": 16.35406581347866
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.118021937826218
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": -7.611128605515098,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -77.77777777777779,
        "score": -7.788527293425895
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 45.204629089096905,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 18.134212470542753
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 45.204629089096905
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 11.222937880038742,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -33.333333333333336,
        "score": 37.177585569940796
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 11.222937880038742
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 27.98629504757422,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 31.784972099244424
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 27.98629504757422
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -25.92255453318933,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -33.333333333333336,
        "score": -68.96263215302712
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 20.969729542778605,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 25.37417462273358
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 20.969729542778605
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -20.360993700444038,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": 22.54071227845883
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -2.256037962010261,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -19.999999999999996,
        "score": 17.398552780635878
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 3.402882822738425,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0,
        "score": 0.1444086395214872
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 3.402882822738425
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 5.5660152900999575,
       "cc_by_type": {
        "choice": 41.860465116279066,
        "noul": -11.111111111111116,
        "score": -14.05130813486808
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 5.5660152900999575
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -12.359465939103076,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": -4.820333301180191
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 14.8266510341226,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -60,
        "score": 27.512582660909047
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 14.8266510341226
      },
      "model_routing": {
       "accuracy": 0.5852534562211982,
       "cc": 32.503636070558116,
       "cc_by_type": {
        "choice": 54.41558441558442,
        "noul": 18.181818181818187,
        "score": 24.91350561427175
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 32.503636070558116
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 26.903819630623776,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 0,
        "score": 26.77671800895386
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 26.903819630623776
      },
      "recruiting": {
       "accuracy": 0.5,
       "cc": 5.445115065352193,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": -33.333333333333336,
        "score": -0.19204570459336878
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 5.445115065352193
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -8.059756459304968,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": -15.06596888530407
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 23.630586887314262,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 47.70335486484133
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 23.630586887314262
      },
      "search_retrieval": {
       "accuracy": 0.625,
       "cc": 48.89395421647513,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 50,
        "score": 28.710332400315085
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 48.89395421647513
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      73.63938162121107,
      82.2058545153825
     ],
     "I": [
      16.166290591940747,
      23.47078252757484
     ],
     "capability": [
      46.01620516917586,
      52.03886983180177
     ],
     "composite": {
      "A": [
       3.909429912465545,
       10.019247475616854
      ],
      "B": [
       3.09621130589029,
       8.449477612845056
      ],
      "C": [
       2.714881883656628,
       6.957810746956148
      ]
     },
     "seed": 53
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.60239191738147,
     "capability": 49.91717019137964,
     "intelligence": 20.23194846537782
    },
    "per_type": {
     "choice": {
      "accuracy": 0.56,
      "calibration": 75.04985336107308,
      "cc": 43.1676840352858,
      "cc_by_type": {
       "choice": 43.1676840352858
      },
      "cc_tiered": 43.30147294542593,
      "cc_tiered_open": 44.315480566561476,
      "cc_tiered_sealed": 42.28746532429039,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 43.1676840352858
     },
     "noul": {
      "accuracy": 0.424,
      "calibration": 77.60729296250656,
      "cc": -15.200000000000003,
      "cc_by_type": {
       "noul": -15.200000000000003
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4,
      "calibration": 86.15002942856476,
      "cc": 13.212269569733326,
      "cc_by_type": {
       "score": 13.212269569733326
      },
      "cc_tiered": 17.394372450707532,
      "cc_tiered_open": 25.784850930770162,
      "cc_tiered_sealed": 9.003893970644905,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 13.212269569733326
     }
    },
    "raw_on_A": {
     "calibration": 77.01332672896311,
     "capability": 47.71286883151212,
     "intelligence": 18.412410934061132
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.009092360003425,
    "Emphasis on Accuracy 60:20:20": 4.533120029666712,
    "Emphasis on Cost 20:20:60": 7.449403907098574,
    "Emphasis on Speed 20:60:20": 6.924924518443358,
    "Intelligence only": 3.3126314570949202
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 86.63426567315432,
    "cost": 60.74282184772803,
    "intelligence": 41.405096245385266,
    "speed": 88.52559981079375
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08196727615205182,
      "ece_score": 83.60654476958965,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 83.60654476958965,
      "score_before_time_limit": 83.60654476958965,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.42666666666666664,
      "brier": 0.11036389023557593,
      "ece": 0.06483675752018682,
      "n": 375,
      "score": 87.03264849596263,
      "score_before_time_limit": 87.03264849596263,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.5255555555555554,
      "n": 375,
      "nrps": 0.08473015745164805,
      "score": 89.26360375391067,
      "score_before_time_limit": 89.26360375391067,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0649988837350692
     }
    },
    "score": 86.63426567315432
   },
   "capability": 64.01968095926979,
   "composite_ci95": {
    "A": [
     32.26129596334245,
     56.1695300004645
    ],
    "B": [
     28.703385183144004,
     51.85751081932692
    ],
    "C": [
     22.403677752321148,
     39.00661805587812
    ]
   },
   "cost": {
    "basis": "Bespoke Labs public list price, USD 0.04/M input, output free (fetched 16 Oct 2026): Bespoke's Nimble pricing page (docs.bespokelabs.ai/nimble/pricing, saved copy sha256 03b06261...) says \"Nimble charges for input tokens only. Output tokens are free.\" and lists nimble-v4 at USD 0.04 per 1M input tokens. Counted as the manufacturer's standard, non-promotional launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap and d1-ultra), through a scoped re-freeze of the frozen pricing module that adds exactly three keys (sage-1.4.0, nimble-v4, nimble-v4-max) to its launch-list-price set and nothing else, with a zero-difference proof for every other registry entry and the independent release review; it is the separate pricing change CoS deputy decision #19084 made the cost axis wait for. Free credit for new accounts is a trial credit and is not used: the row is priced at the published tariff, never as cost 0. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (1,935,351 input tokens over all 1,500 answered items; the 3,000 reported output tokens are free under the tariff), divided by the 1,485 items of the common cost basis the scorer uses. USD 0.020350 per 1,000 decisions on the common basis (USD 0.051609 over all 1,500 items), 0.315x the official cost cap USD 0.064595. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.020350 per 1,000 decisions (0.315x the official cost cap USD 0.064595), against USD 0.051609 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05160936000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020350383838383838
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble-v4 (Bespoke Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.bespokelabs.ai /v1/systemone API, model `nimble-v4`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `bespoke` before the first request (CoS deputy #18998/#19084, custodian #18999; every sealed item of this draw was already retired). Native probabilities on all three question types. Hosting implementation and deployed weights were not independently inspected. Measured as our own add after Bespoke Labs' public announcement of 13 Oct 2026 (no paid order); key on Benchmark Heaven's own account with Bespoke's free credit.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 46.227471634307484,
    "I_sealed": 36.582720856463055,
    "base": 41.405096245385266,
    "excess": -0.19477599251128197,
    "gap": 9.644750777844429,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.38459954164753,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 13.326432022084196,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": 0.0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 50.97138333919072,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.922961492763804,
       "hard": 58.84686036423604,
       "judge": 45.799744250773585,
       "standard": 43.502098844939226
      }
     },
     "sealed|choice": {
      "cc": 66.57364668901572,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 82.84625158831004,
       "hard": 66.69365097721753,
       "judge": 57.68854064642509,
       "standard": 71.52499472685088
      }
     },
     "sealed|noul": {
      "cc": 3.5399803490385073,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 3.6363636363636376,
       "judge": -19.999999999999996,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 39.63453553133493,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 63.47008501291492,
       "hard": 32.35700189778908,
       "judge": 34.84728157786855,
       "standard": 49.4527089878362
      }
     }
    }
   },
   "jevbench_score": 43.22874272615112,
   "key": "nimble-v4",
   "last_measured_on": "2026-10-14",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "listing_note": "Bespoke told us in a chat on 16 Oct 2026 that Nimble-v4 is deprecated; its pricing page still lists it (only nimble-v3 is marked deprecated there). The page lists it at USD 0.04 per million input tokens. Measured values, price and rank unchanged (JevBench v1.6.26 notes).",
   "measured_in": "v1.6.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.bespokelabs.ai/v1/systemone#nimble-v4",
   "noul_decisive": {
    "acc_among_decisive": 0.9627906976744186,
    "decisive": 215,
    "decisive_rate": 0.5733333333333334,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 26,
    "axes": {
     "calibration": 86.63426567315432,
     "cost": 60.74282184772803,
     "intelligence": 41.405096245385266,
     "speed": 88.39856712562789
    },
    "board_release": "v1.6.43",
    "capability": 64.01968095926979,
    "capability_api_board_rank": 22,
    "capability_open_board_rank": null,
    "capability_rank": 63,
    "composite_ci95": {
     "A": [
      32.253505531154396,
      56.15459520353683
     ],
     "B": [
      28.698448652413568,
      51.84732628344259
     ],
     "C": [
      22.39826772996833,
      38.9962466691228
     ]
    },
    "jevbench_score": 43.21768647452467,
    "measured_in": "v1.6.18",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 74,
    "ranks": {
     "A": 74,
     "B": 78,
     "C": 78,
     "capability": 105
    },
    "scores": {
     "A": 43.21768647452467,
     "B": 39.13164408365893,
     "C": 30.01228227397547
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.23237984799197875,
     "p50_s_raw": 0.23237984799197875,
     "p95_s_adjusted": 0.6222212287582807,
     "p95_s_raw": 0.6222212287582807
    },
    "views": {
     "Balanced 33:33:33": 39.618380485230354,
     "Emphasis on Accuracy 60:20:20": 34.208925086970844,
     "Emphasis on Cost 20:20:60": 40.408487852374435,
     "Emphasis on Speed 20:60:20": 45.99178065109866,
     "Intelligence only": 28.393660603216478
    }
   },
   "rank": 84,
   "ranked": true,
   "ranks": {
    "A": 84,
    "B": 89,
    "C": 90,
    "capability": 122
   },
   "repo": "https://bespokelabs.ai",
   "repriced_in": "v1.6.23",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 43.22874272615112,
    "B": 39.13889512595354,
    "C": 30.019960226493836
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2323382355098147,
    "p50_s_raw": 0.2323382355098147,
    "p95_s_adjusted": 0.6043928697079537,
    "p95_s_raw": 0.6043928697079537
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 62.87504268080306,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 5.291794709075859
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 62.87504268080306
      },
      "calibration_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 36.71840534927043,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 36.00000000000001,
        "score": 21.875258465626757
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 36.71840534927043
      },
      "composite_gen": {
       "accuracy": 0.6831683168316832,
       "cc": 38.05576935546986,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": -25.92592592592593,
        "score": 46.597299032985916
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 38.05576935546986
      },
      "fairness": {
       "accuracy": 0.8333333333333334,
       "cc": 68.76166602198055,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 72.95166473260831
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 68.76166602198055
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 22.243715441838727,
       "cc_by_type": {
        "choice": 46.58325312800771,
        "noul": -29.729729729729726,
        "score": 49.877622927238185
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 22.243715441838727
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 62.50547658592752,
       "cc_by_type": {
        "choice": 66.71656247248393,
        "noul": 62.88659793814433,
        "score": 57.91326934715432
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.50547658592752
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": 4.871919127970649,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": 0.0,
        "score": -46.67456519673321
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 4.871919127970649
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -12.631364067994236,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75.0,
        "score": -13.409556121508492
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6581818181818182,
       "cc": 36.79679245547057,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": -11.111111111111116,
        "score": 47.91166128482639
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 36.79679245547057
      },
      "negation_pair_gen": {
       "accuracy": 0.627906976744186,
       "cc": 17.910040456548206,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": -46.666666666666664,
        "score": 7.063454702977956
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 17.910040456548206
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 71.46060305149378,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100.0,
        "score": 44.41183918451137
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 71.46060305149378
      },
      "usecase": {
       "accuracy": 0.5727272727272728,
       "cc": 30.241095001794505,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": -8.333333333333337,
        "score": 37.04681441714821
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 30.241095001794505
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": -0.4063065270671255,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 0.0,
        "score": -63.12368148596328
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 17.33676891316524,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 6.0809672617691435
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 17.33676891316524
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 72.84971754245485,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 67.44295793709905
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.84971754245485
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 33.173708716426326,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 99.70734588857135
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 33.173708716426326
      },
      "en": {
       "accuracy": 0.6637706342311034,
       "cc": 42.06591187806426,
       "cc_by_type": {
        "choice": 68.58715417190416,
        "noul": 13.669064748201443,
        "score": 43.94151671408719
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 42.06591187806426
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 67.31271336713421,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 11.726068323262318
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.31271336713421
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 25.707953232257534,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 43.24407827600757
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 25.707953232257534
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 72.26504569240879,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100.0,
        "score": 80.32454884193226
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 72.26504569240879
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 22.236011151779042,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25.0,
        "score": 25.22758652796282
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 22.236011151779042
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 71.53128312800176,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 28.623467466624174
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.53128312800176
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 14.784765883400851,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": 21.542542288804622
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 14.784765883400851
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 10.452983327342158,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 31.358949982026463
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 10.452983327342158
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.30841577705657,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 98.59191399783639
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.30841577705657
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -6.49782092272594,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 32.45458917987413
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 38.61167387134184,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 51.06989254081351
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 38.61167387134184
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 50.52489539329753,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 19.999999999999996,
        "score": 63.39554652167219
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 50.52489539329753
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 13.221329392617008,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 36.25973285870208
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 13.221329392617008
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 37.8682340701448,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50.0,
        "score": -14.65616735478299
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 37.8682340701448
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.86133582192467,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 42.584007465774
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.86133582192467
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 11.860785606137426,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -31.08430984825439
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 11.860785606137426
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7626666666666667,
       "cc": 69.34499320691174,
       "cc_by_type": {
        "choice": 69.34499320691174
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.34499320691174
      },
      "noul": {
       "accuracy": 0.552,
       "cc": 10.40000000000001,
       "cc_by_type": {
        "noul": 10.40000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 10.40000000000001
      },
      "score": {
       "accuracy": 0.576,
       "cc": 43.476768622445704,
       "cc_by_type": {
        "score": 43.476768622445704
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 43.476768622445704
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 2.2786515839975863,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -75.0,
        "score": 13.835954751992762
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 2.2786515839975863
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 16.217833358888537,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -11.111111111111116,
        "score": -2.968308066881664
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 16.217833358888537
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 30.601303418460095,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 40.58439806025833
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.601303418460095
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -29.26143997524909,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100.0,
        "score": 5.782931536241042
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 22.322772125960967,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 58.27266420396986
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.322772125960967
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -22.96464494240405,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -26.75557004733793
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 17.40028973466517,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 14.665855198393274
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 17.40028973466517
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 2.0356283791569916,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -100.0,
        "score": 37.465770154892574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 2.0356283791569916
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": 5.2997131864851355,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 38.399139559455406
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 5.2997131864851355
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 41.33594739962141,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 33.33333333333333,
        "score": 29.218406081805014
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 41.33594739962141
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 24.531777745700577,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 18.556573547179266
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 24.531777745700577
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 12.053690294805428,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -33.333333333333336,
        "score": 1.7524687338786538
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.053690294805428
      },
      "llm_guardrails": {
       "accuracy": 0.7037037037037037,
       "cc": 41.77608289103801,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 56.84456345238465
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 41.77608289103801
      },
      "model_routing": {
       "accuracy": 0.815668202764977,
       "cc": 68.58454886246625,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 81.81818181818181,
        "score": 44.974425808178
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 68.58454886246625
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 8.881552486989458,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 61.193601798780286
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 8.881552486989458
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 50.38160882884504,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 51.23767699721296
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.38160882884504
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -0.19458016484627372,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": 8.529559998072012
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 13.047825366682751,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 30.447823926135207
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 13.047825366682751
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 93.19059202541666,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 79.57177607624996
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 93.19059202541666
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.37908894972551,
      87.49271267670296
     ],
     "I": [
      36.80192764711807,
      46.306385990881296
     ],
     "capability": [
      60.20595188468623,
      65.72729625625716
     ],
     "composite": {
      "A": [
       32.26129596334245,
       56.1695300004645
      ],
      "B": [
       28.703385183144004,
       51.85751081932692
      ],
      "C": [
       22.403677752321148,
       39.00661805587812
      ]
     },
     "seed": 54
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.63426567315432,
     "capability": 64.01968095926979,
     "intelligence": 41.405096245385266
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7626666666666667,
      "calibration": 83.60654476958965,
      "cc": 69.34499320691174,
      "cc_by_type": {
       "choice": 69.34499320691174
      },
      "cc_tiered": 70.47912311533162,
      "cc_tiered_open": 74.38459954164753,
      "cc_tiered_sealed": 66.57364668901572,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.34499320691174
     },
     "noul": {
      "accuracy": 0.552,
      "calibration": 87.03264849596263,
      "cc": 10.40000000000001,
      "cc_by_type": {
       "noul": 10.40000000000001
      },
      "cc_tiered": 8.433206185561351,
      "cc_tiered_open": 13.326432022084196,
      "cc_tiered_sealed": 3.5399803490385073,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 10.40000000000001
     },
     "score": {
      "accuracy": 0.576,
      "calibration": 89.26360375391067,
      "cc": 43.476768622445704,
      "cc_by_type": {
       "score": 43.476768622445704
      },
      "cc_tiered": 45.30295943526283,
      "cc_tiered_open": 50.97138333919072,
      "cc_tiered_sealed": 39.63453553133493,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.476768622445704
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 39.630769642234846,
    "Emphasis on Accuracy 60:20:20": 34.21446640379054,
    "Emphasis on Cost 20:20:60": 40.41621985641152,
    "Emphasis on Speed 20:60:20": 46.021843440308366,
    "Intelligence only": 28.393660603216478
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 91.24998737212707,
    "cost": 60.74282184772803,
    "intelligence": 68.18491883484498,
    "speed": 87.92708417376494
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04168013967750483,
      "ece_score": 91.66397206449903,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 91.66397206449903,
      "score_before_time_limit": 91.66397206449903,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.21599999999999997,
      "brier": 0.060518846834681644,
      "ece": 0.04479113991582226,
      "n": 375,
      "score": 91.04177201683554,
      "score_before_time_limit": 91.04177201683554,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9973333333333333,
      "interval90_width": 0.3084444444444445,
      "n": 375,
      "nrps": 0.04645325205782863,
      "score": 91.04421803504661,
      "score_before_time_limit": 91.04421803504661,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06633119362061955
     }
    },
    "score": 91.24998737212707
   },
   "capability": 79.71745310348602,
   "composite_ci95": {
    "A": [
     72.72057952799157,
     75.71656715778371
    ],
    "B": [
     70.5838602411584,
     74.87875387318591
    ],
    "C": [
     72.72057952799157,
     75.71656715778371
    ]
   },
   "cost": {
    "basis": "Bespoke Labs public list price, USD 0.04/M input, output free: Bespoke's Nimble pricing page (docs.bespokelabs.ai/nimble/pricing) says \"Nimble charges for input tokens only. Output tokens are free.\" and, in the copy fetched 16 Oct 2026 at about 21:40Z (saved copy sha256 4d46184d...), lists nimble-v4-max at USD 0.04 per 1M input tokens. The copy of the same page fetched 16 Oct 2026 at 15:58Z (saved copy sha256 03b06261...), from which v1.6.23 first priced this row, listed USD 0.09; v1.6.26 re-prices the row to the new public tariff through a scoped re-freeze of the frozen pricing module that changes the documented tariff of this one key and nothing else (no key joins or leaves its launch-list-price set; zero difference for every other registry entry). This is the current public list price on the vendor's pricing page (changed 16 Oct 2026; earlier copy listed USD 0.09), counted under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap and d1-ultra). Free credit for new accounts is a trial credit and is not used: the row is priced at the published tariff, never as cost 0. No new measurement: ESTIMATE in the scorer's sense, the tariff x this row's OWN provider-reported tokens from its v1.6.18 measurement (1,935,351 input tokens over all 1,500 answered items; the 3,000 reported output tokens are free under the tariff), divided by the 1,485 items of the common cost basis the scorer uses. USD 0.020350 per 1,000 decisions on the common basis (USD 0.051609 over all 1,500 items), 0.315x the official cost cap USD 0.064595 (at the earlier USD 0.09 tariff it was over the cap). Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.020350 per 1,000 decisions (0.315x the official cost cap USD 0.064595), against USD 0.051609 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05160936000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020350383838383838
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble-v4-max (Bespoke Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.bespokelabs.ai /v1/systemone API, model `nimble-v4-max`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `bespoke` before the first request (CoS deputy #18998/#19084, custodian #18999; every sealed item of this draw was already retired). Native probabilities on all three question types. Hosting implementation and deployed weights were not independently inspected. Measured as our own add after Bespoke Labs' public announcement of 13 Oct 2026 (no paid order); key on Benchmark Heaven's own account with Bespoke's free credit.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.20451669265262,
    "I_sealed": 64.16532097703733,
    "base": 68.18491883484498,
    "excess": -1.8003310547404183,
    "gap": 8.039195715615293,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 86.67735092678937,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 84.90091223655237,
       "judge": 92.089650626236,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 63.948240165631475,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 65.98795898553702,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.01753536790163,
       "hard": 72.99899229354084,
       "judge": 67.76239404368843,
       "standard": 48.28945159111995
      }
     },
     "sealed|choice": {
      "cc": 80.44759269360293,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 74.49982652943217,
       "judge": 81.48873653281098,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 46.774787744422326,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 43.636363636363626,
       "judge": 38.46153846153846,
       "standard": 49.25373134328359
      }
     },
     "sealed|score": {
      "cc": 65.27358249308672,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.33978396612804,
       "hard": 64.40573294358967,
       "judge": 63.511594612062325,
       "standard": 62.619162677096796
      }
     }
    }
   },
   "jevbench_score": 74.82115516763538,
   "key": "nimble-v4-max",
   "last_measured_on": "2026-10-14",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.bespokelabs.ai/v1/systemone#nimble-v4-max",
   "noul_decisive": {
    "acc_among_decisive": 0.9795918367346939,
    "decisive": 294,
    "decisive_rate": 0.784,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 5,
    "axes": {
     "calibration": 91.24998737212707,
     "cost": 60.74282184772803,
     "intelligence": 68.18491883484498,
     "speed": 91.45091520080334
    },
    "board_release": "v1.6.43",
    "capability": 79.71745310348602,
    "capability_api_board_rank": 7,
    "capability_open_board_rank": null,
    "capability_rank": 12,
    "composite_ci95": {
     "A": [
      73.30460656936545,
      76.34991725686463
     ],
     "B": [
      71.02324021608213,
      75.3734188553226
     ],
     "C": [
      73.30460656936545,
      76.34991725686463
     ]
    },
    "jevbench_score": 75.4395528754179,
    "measured_in": "v1.6.18",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 6,
    "ranks": {
     "A": 6,
     "B": 9,
     "C": 6,
     "capability": 24
    },
    "scores": {
     "A": 75.4395528754179,
     "B": 73.8677004961083,
     "C": 75.4395528754179
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.254514531508903,
     "p50_s_raw": 0.254514531508903,
     "p95_s_adjusted": 0.28131694759504167,
     "p95_s_raw": 0.28131694759504167
    },
    "views": {
     "Balanced 33:33:33": 71.32043947749163,
     "Emphasis on Accuracy 60:20:20": 70.03225099016612,
     "Emphasis on Cost 20:20:60": 66.67610875958445,
     "Emphasis on Speed 20:60:20": 78.20646502285324,
     "Intelligence only": 68.18491883484498
    }
   },
   "rank": 8,
   "ranked": true,
   "ranks": {
    "A": 8,
    "B": 11,
    "C": 8,
    "capability": 30
   },
   "repo": "https://bespokelabs.ai",
   "repriced_in": "v1.6.26",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.82115516763538,
    "B": 73.39253944111499,
    "C": 74.82115516763538
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2574840509914793,
    "p50_s_raw": 0.2574840509914793,
    "p95_s_adjusted": 0.6259523141590762,
    "p95_s_raw": 0.6259523141590762
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 77.93302244321468,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 42.13240066297739
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 77.93302244321468
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 48.46720156444257,
       "cc_by_type": {
        "choice": 61.82396606574761,
        "noul": 28.000000000000004,
        "score": 55.57763862758007
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 48.46720156444257
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 85.97656131741992,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 71.8411564003176
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.97656131741992
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.3952829117887,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 96.18584873536608
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.3952829117887
      },
      "jaggedness": {
       "accuracy": 0.7444444444444445,
       "cc": 52.85440527377147,
       "cc_by_type": {
        "choice": 68.23869104908566,
        "noul": 13.513513513513509,
        "score": 76.81101125871524
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 52.85440527377147
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.04124579094008,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 93.81443298969072,
        "score": 60.85654397104885
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.04124579094008
      },
      "long_state": {
       "accuracy": 0.6,
       "cc": 23.69771087111538,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": 0.0,
        "score": 9.802810032700982
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 23.69771087111538
      },
      "ml_usecase": {
       "accuracy": 0.5925925925925926,
       "cc": 18.542786785637407,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -25.0,
        "score": 30.11289643938645
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 18.542786785637407
      },
      "multilingual": {
       "accuracy": 0.7345454545454545,
       "cc": 54.53849518745235,
       "cc_by_type": {
        "choice": 76.5242908379524,
        "noul": 25.0,
        "score": 62.091194724404616
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 54.53849518745235
      },
      "negation_pair_gen": {
       "accuracy": 0.9302325581395349,
       "cc": 78.91939008748496,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 70.09150359578821
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.91939008748496
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 82.73195100155357,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 63.21086801967573
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.73195100155357
      },
      "usecase": {
       "accuracy": 0.7454545454545455,
       "cc": 56.032678027680525,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 29.166666666666675,
        "score": 64.66666153402196
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 56.032678027680525
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 58.740127058626875,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": -4.73199977650034
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 58.740127058626875
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 41.85206203924473,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 12.960179973340946
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 41.85206203924473
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 82.27735534531733,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100.0,
        "score": 78.69047311559798
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 82.27735534531733
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 61.060090115548206,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0.0,
        "score": 99.94004688295746
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 61.060090115548206
      },
      "en": {
       "accuracy": 0.8392701998262381,
       "cc": 71.15704384183026,
       "cc_by_type": {
        "choice": 83.52473120904064,
        "noul": 60.431654676258994,
        "score": 69.51474564019114
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.15704384183026
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 69.66571860471659,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 18.78508403600947
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.66571860471659
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 76.57541393410838,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 96.39290846899185
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 76.57541393410838
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 89.21722025064541,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 88.8281313401715
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 89.21722025064541
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 34.367393690812165,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 36.62173414506219
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 34.367393690812165
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 70.95270534511762,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 26.887734117971718
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 70.95270534511762
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 27.327090862326045,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0.0,
        "score": 10.575394906279168
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 27.327090862326045
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 3.596678785557382,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 55.23448080111659
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 3.596678785557382
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.40723626765454,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 98.88837546963032
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.40723626765454
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -26.53328200822546,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": -27.651794076624437
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25.0
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 73.1717555920901,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 67.13270223966425
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 73.1717555920901
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 65.04851477203407,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 66.96640465788181
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.04851477203407
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 40.482391585259116,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 52.51100454301141
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.482391585259116
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 60.52275499920906,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 31.568264997627193
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 60.52275499920906
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.19627963052231,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 81.10496792382497
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.19627963052231
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 33.346706726269964,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 33.373453512143236
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.346706726269964
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8613333333333333,
       "cc": 82.08920951415067,
       "cc_by_type": {
        "choice": 82.08920951415067
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.08920951415067
      },
      "noul": {
       "accuracy": 0.768,
       "cc": 53.6,
       "cc_by_type": {
        "noul": 53.6
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 53.6
      },
      "score": {
       "accuracy": 0.776,
       "cc": 66.16378952885489,
       "cc_by_type": {
        "score": 66.16378952885489
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 66.16378952885489
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 11.284240243103335,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -75.0,
        "score": 40.85272072931001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 11.284240243103335
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 81.22933598842444,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 62.321548337944144
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 81.22933598842444
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 21.699250411759156,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 13.878239040155515
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 21.699250411759156
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -20.13552623419365,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": -45.20190034000786
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.00228808667219,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 82.32570483972668
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.00228808667219
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": -25.10701220211857,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": -196.07575358748778
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 31.913037340162806,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -42.85714285714286,
        "score": 48.400176446258726
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 31.913037340162806
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 52.57354917823071,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 57.72064753469211
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 52.57354917823071
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 15.471706603552159,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 68.91511981065648
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 15.471706603552159
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 64.00816563621142,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 63.901727458241695
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 64.00816563621142
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 57.96643471148216,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 52.19387777785735
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 57.96643471148216
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 19.212456270641866,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 21.07822902697937
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 19.212456270641866
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 55.60780315989738,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 58.33972425896277
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.60780315989738
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 79.1817133930041,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 93.93939393939394,
        "score": 49.4499020837742
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.1817133930041
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 15.930238569976515,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 82.33966004774146
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 15.930238569976515
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 57.68500793305302,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.1478743098369
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.68500793305302
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 36.52975459820843,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 24.367588917777994
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 36.52975459820843
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 6.286419441145905,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -66.66666666666667,
        "score": 57.98969310604642
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 6.286419441145905
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.53112849752597,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.59338549257794
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.53112849752597
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.9087720581256,
      91.64106229553519
     ],
     "I": [
      62.909489146085264,
      71.90798251065188
     ],
     "capability": [
      75.86144294449328,
      81.13427659789494
     ],
     "composite": {
      "A": [
       72.72057952799157,
       75.71656715778371
      ],
      "B": [
       70.5838602411584,
       74.87875387318591
      ],
      "C": [
       72.72057952799157,
       75.71656715778371
      ]
     },
     "seed": 55
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.24998737212707,
     "capability": 79.71745310348602,
     "intelligence": 68.18491883484498
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8613333333333333,
      "calibration": 91.66397206449903,
      "cc": 82.08920951415067,
      "cc_by_type": {
       "choice": 82.08920951415067
      },
      "cc_tiered": 83.56247181019614,
      "cc_tiered_open": 86.67735092678937,
      "cc_tiered_sealed": 80.44759269360293,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.08920951415067
     },
     "noul": {
      "accuracy": 0.768,
      "calibration": 91.04177201683554,
      "cc": 53.6,
      "cc_by_type": {
       "noul": 53.6
      },
      "cc_tiered": 55.3615139550269,
      "cc_tiered_open": 63.948240165631475,
      "cc_tiered_sealed": 46.774787744422326,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.6
     },
     "score": {
      "accuracy": 0.776,
      "calibration": 91.04421803504661,
      "cc": 66.16378952885489,
      "cc_by_type": {
       "score": 66.16378952885489
      },
      "cc_tiered": 65.63077073931187,
      "cc_tiered_open": 65.98795898553702,
      "cc_tiered_sealed": 65.27358249308672,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 66.16378952885489
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.58506270446225,
    "Emphasis on Accuracy 60:20:20": 69.60500998393475,
    "Emphasis on Cost 20:20:60": 66.28872247626451,
    "Emphasis on Speed 20:60:20": 76.63066590299633,
    "Intelligence only": 68.18491883484498
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "BlackdromeAILabs",
   "axes": {
    "calibration": 83.63708212673413,
    "cost": 61.95354260266235,
    "intelligence": 25.051442411072966,
    "speed": 93.07972984765502
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07120275435939666,
      "ece_score": 85.75944912812066,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 85.75944912812066,
      "score_before_time_limit": 85.75944912812066,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.4906666666666667,
      "brier": 0.17048542395103217,
      "ece": 0.10058815024855235,
      "n": 375,
      "score": 79.88236995028953,
      "score_before_time_limit": 79.88236995028953,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9013333333333333,
      "interval90_width": 0.5231111111111111,
      "n": 375,
      "nrps": 0.13612432038172226,
      "score": 85.26942730179223,
      "score_before_time_limit": 85.26942730179223,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07924356679121652
     }
    },
    "score": 83.63708212673413
   },
   "capability": 54.34426226890355,
   "composite_ci95": {
    "A": [
     7.638660411095513,
     17.44367243497911
    ],
    "B": [
     6.146196710787558,
     14.77470853361324
    ],
    "C": [
     5.304625285482995,
     12.113661413179937
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-4B-Base -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835, the module this row was first priced with (the later re-freezes only add hosted-API launch-list keys), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). Tokens as reported by the author's server (output_tokens 0, nothing generated). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.018545 per 1,000 decisions (0.287x the official cost cap USD 0.064595), against USD 0.019615 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.019615120000000003
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01854450505050505
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Noma (Blackdrome AI Labs, Qwen3.5-4B-Base derivative)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20150, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own `noma serve` with the default fast path (128-token buckets + CUDA graphs).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 29.756564202105558,
    "I_sealed": 20.346320620040373,
    "base": 25.051442411072966,
    "excess": -0.42928318829052614,
    "gap": 9.410243582065185,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 53.03396058840887,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 41.491034916640444,
       "judge": 64.40342781806197,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 36.23573201790782,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.562287908823535,
       "hard": 33.84744031581148,
       "judge": 44.6284633982232,
       "standard": 21.759940406169587
      }
     },
     "sealed|choice": {
      "cc": 48.83838252279794,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 43.79553602405458,
       "judge": 50.6366307541626,
       "standard": 46.84665682345498
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 10.344827586206895,
       "hard": -12.727272727272732,
       "judge": -38.46153846153846,
       "standard": 4.477611940298498
      }
     },
     "sealed|score": {
      "cc": 12.200579337323186,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 26.122391760234898,
       "hard": 11.720567435668894,
       "judge": 1.0388793398912077,
       "standard": 22.942246925323882
      }
     }
    }
   },
   "jevbench_score": 12.749286998918699,
   "key": "noma",
   "last_measured_on": "2026-10-16",
   "licence": "MPL-2.0 (weights and code)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "BlackdromeAILabs/noma@fac0429688a137ce451a27ad34d05e9fd47fb624",
   "noul_decisive": {
    "acc_among_decisive": 0.8848167539267016,
    "decisive": 191,
    "decisive_rate": 0.5093333333333333,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.63708212673413,
     "cost": 61.95354260266235,
     "intelligence": 25.051442411072966,
     "speed": 93.34244428975083
    },
    "board_release": "v1.6.43",
    "capability": 54.34426226890355,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 85,
    "capability_rank": 111,
    "composite_ci95": {
     "A": [
      7.641283470998092,
      17.450802665135193
     ],
     "B": [
      6.147554589025266,
      14.778800854363228
     ],
     "C": [
      5.306446854859787,
      12.11861296189944
     ]
    },
    "jevbench_score": 12.754183690755768,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 124,
    "rank": 163,
    "ranks": {
     "A": 163,
     "B": 166,
     "C": 164,
     "capability": 166
    },
    "scores": {
     "A": 12.754183690755768,
     "B": 10.578899893750625,
     "C": 8.857072007469284
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2009935579902958,
     "p50_s_raw": 0.025496778995147906,
     "p95_s_adjusted": 0.23044825975957792,
     "p95_s_raw": 0.04022412987978896
    },
    "views": {
     "Balanced 33:33:33": 11.278487206830306,
     "Emphasis on Accuracy 60:20:20": 8.561265131134954,
     "Emphasis on Cost 20:20:60": 12.67130479105097,
     "Emphasis on Speed 20:60:20": 14.230928812591463,
     "Intelligence only": 6.2886612524076995
    }
   },
   "rank": 184,
   "ranked": true,
   "ranks": {
    "A": 184,
    "B": 188,
    "C": 186,
    "capability": 186
   },
   "repo": "https://huggingface.co/BlackdromeAILabs/noma",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 12.749286998918699,
    "B": 10.576204485571711,
    "C": 8.853671527026874
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20187975100125186,
    "p50_s_raw": 0.02593987550062593,
    "p95_s_adjusted": 0.24374418011284432,
    "p95_s_raw": 0.046872090056422164
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.45454545454545453,
       "cc": 17.41950772702694,
       "cc_by_type": {
        "choice": 25.0,
        "noul": 50.0,
        "score": -22.741476818919182
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 17.41950772702694
      },
      "calibration_gen": {
       "accuracy": 0.2871287128712871,
       "cc": -16.632615001550132,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -43.99999999999999,
        "score": -20.00176865258254
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 7.918370014291946,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -18.518518518518523,
        "score": -25.206046235353607
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 7.918370014291946
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 55.05881191793984,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 45.17643575381953
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 55.05881191793984
      },
      "jaggedness": {
       "accuracy": 0.4388888888888889,
       "cc": -1.015847243318935,
       "cc_by_type": {
        "choice": 33.58999037536093,
        "noul": -40.54054054054054,
        "score": 3.90300843522281
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.7188498402555911,
       "cc": 52.54172250879653,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 46.39175257731958,
        "score": 46.10177807313447
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 52.54172250879653
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -29.786333031651964,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0.0,
        "score": -60.326741030439756
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -27.07773849436275,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100.0,
        "score": 5.364722661241627
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5163636363636364,
       "cc": 17.772590134559422,
       "cc_by_type": {
        "choice": 49.135963482230196,
        "noul": -30.555555555555557,
        "score": 34.73736247700363
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 17.772590134559422
      },
      "negation_pair_gen": {
       "accuracy": 0.4186046511627907,
       "cc": -14.28969892052313,
       "cc_by_type": {
        "choice": 43.33333333333332,
        "noul": -53.333333333333336,
        "score": -32.869096761569374
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.7708333333333334,
       "cc": 70.70572569906791,
       "cc_by_type": {
        "choice": 70.70572569906791
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 70.70572569906791
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 70.18476184672892,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 80.0,
        "score": 66.59032157622278
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.18476184672892
      },
      "usecase": {
       "accuracy": 0.4409090909090909,
       "cc": 10.091614151059419,
       "cc_by_type": {
        "choice": 44.85294117647061,
        "noul": -37.5,
        "score": 22.921901276707647
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 10.091614151059419
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -27.048739826395394,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50.0,
        "score": -16.86050519347191
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.2857142857142857,
       "cc": 47.368421052631575,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 47.368421052631575
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 10.925357399486423,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": -13.153267279267311
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 10.925357399486423
      },
      "de": {
       "accuracy": 0.59375,
       "cc": 30.560566461628643,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 0.0,
        "score": 31.504708234443456
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 30.560566461628643
      },
      "el": {
       "accuracy": 0.6,
       "cc": 44.32658194414518,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 99.83263223839458
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.32658194414518
      },
      "en": {
       "accuracy": 0.5369244135534318,
       "cc": 19.628069056743954,
       "cc_by_type": {
        "choice": 49.69551262493742,
        "noul": -5.755395683453235,
        "score": 14.944090228747676
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 19.628069056743954
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 57.38676236262902,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 75.0,
        "score": 16.736143531606473
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 57.38676236262902
      },
      "fi": {
       "accuracy": 0.375,
       "cc": -5.7500144985855615,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -100.0,
        "score": 15.536841750144958
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 37.1638514750788,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 53.84449560170699
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 37.1638514750788
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 1.2012258532770967,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -75.0,
        "score": 28.883007168769836
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 1.2012258532770967
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 78.87962593695318,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 50.66849589347838
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 78.87962593695318
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 24.526026775767367,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 10.766324965904172
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 24.526026775767367
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 3.947561358412102,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 11.84268407523631
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 3.947561358412102
      },
      "mixed": {
       "accuracy": 0.6428571428571429,
       "cc": 55.142380630907915,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 48.760475226057075
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 55.142380630907915
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -2.1042632801837207,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100.0,
        "score": 71.60928808152676
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 18.96793208728958,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -60.0,
        "score": 39.756102652050714
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 18.96793208728958
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 21.53686440182103,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": -19.999999999999996,
        "score": 48.25231388902231
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 21.53686440182103
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": 0.8117545513000266,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60.0,
        "score": 64.56292322836816
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.8117545513000266
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -8.704425914860943,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0.0,
        "score": -17.41762557066979
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 62.31405497077973,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 51.458293944597244
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.31405497077973
      },
      "zh": {
       "accuracy": 0.4,
       "cc": -15.528851182756002,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": -36.982033774256706
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6133333333333333,
       "cc": 50.0564496067663,
       "cc_by_type": {
        "choice": 50.0564496067663
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.0564496067663
      },
      "noul": {
       "accuracy": 0.45066666666666666,
       "cc": -9.866666666666667,
       "cc_by_type": {
        "noul": -9.866666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.464,
       "cc": 18.86027982285219,
       "cc_by_type": {
        "score": 18.86027982285219
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 18.86027982285219
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -37.645864323073745,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -75.0,
        "score": -57.93759296922125
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 8.8939425965539,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": 15.91578224100746
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 8.8939425965539
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 36.874031663183274,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 59.40258279442787
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 36.874031663183274
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -22.785404404592622,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": 36.906944680958986
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -18.60576964001473,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100.0,
        "score": 2.153705572709419
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -32.66952881898021,
       "cc_by_type": {
        "choice": -0.62893081761006,
        "noul": -66.66666666666667,
        "score": -30.7129889726639
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 10.016228768867249,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 30.889022441055523
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 10.016228768867249
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -20.483787632035405,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 11.719368811210861
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 19.43280212374197,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60.0,
        "score": 59.96507303789258
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 19.43280212374197
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -5.167247968174982,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -33.333333333333336,
        "score": 7.767349600114604
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.44,
       "cc": 14.161840905735474,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -55.55555555555556,
        "score": 21.296892226250353
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 14.161840905735474
      },
      "legal_compliance": {
       "accuracy": 0.1,
       "cc": -45.33045811238148,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -6.959116272628307
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 24.176631357497374,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 38.59515318957466
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 24.176631357497374
      },
      "model_routing": {
       "accuracy": 0.7188940092165899,
       "cc": 49.58116367259822,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 45.45454545454546,
        "score": 31.340893615197263
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 49.58116367259822
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -5.966226657212023,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100.0,
        "score": 51.19920870398773
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 44.43452582725566,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 33.396427992444785
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 44.43452582725566
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -2.43732824862776,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -100.0,
        "score": 7.4663403772694314
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.30434782608695654,
       "cc": -9.883258988665188,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -66.66666666666667,
        "score": 23.973411439801538
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 86.81002782020666,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 71.1062400442499
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 86.81002782020666
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.23201053404956,
      85.02161075963176
     ],
     "I": [
      20.510858836085088,
      28.433139202732264
     ],
     "capability": [
      49.788189765031966,
      55.97490663550985
     ],
     "composite": {
      "A": [
       7.638660411095513,
       17.44367243497911
      ],
      "B": [
       6.146196710787558,
       14.77470853361324
      ],
      "C": [
       5.304625285482995,
       12.113661413179937
      ]
     },
     "seed": 56
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.63708212673413,
     "capability": 54.34426226890355,
     "intelligence": 25.051442411072966
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6133333333333333,
      "calibration": 85.75944912812066,
      "cc": 50.0564496067663,
      "cc_by_type": {
       "choice": 50.0564496067663
      },
      "cc_tiered": 50.93617155560341,
      "cc_tiered_open": 53.03396058840887,
      "cc_tiered_sealed": 48.83838252279794,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.0564496067663
     },
     "noul": {
      "accuracy": 0.45066666666666666,
      "calibration": 79.88236995028953,
      "cc": -9.866666666666667,
      "cc_by_type": {
       "noul": -9.866666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.464,
      "calibration": 85.26942730179223,
      "cc": 18.86027982285219,
      "cc_by_type": {
       "score": 18.86027982285219
      },
      "cc_tiered": 24.218155677615503,
      "cc_tiered_open": 36.23573201790782,
      "cc_tiered_sealed": 12.200579337323186,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 18.86027982285219
     }
    },
    "raw_on_A": {
     "calibration": 81.23569238761198,
     "capability": 52.611491162690655,
     "intelligence": 23.987289937769326
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 11.273382062533422,
    "Emphasis on Accuracy 60:20:20": 8.55949974169183,
    "Emphasis on Cost 20:20:60": 12.667437878810665,
    "Emphasis on Speed 20:60:20": 14.216307192502907,
    "Intelligence only": 6.2886612524076995
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Phocinae / NovelNexus AI (novelnexusai@outlook.com)",
   "axes": {
    "calibration": 56.87058116402116,
    "cost": 95.30667675976518,
    "intelligence": 3.609767391262904,
    "speed": 76.50701878654316
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06613968253968247,
      "ece_score": 86.7720634920635,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.7720634920635,
      "score_before_time_limit": 86.7720634920635,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.9706666666666667,
      "brier": 0.24743832458666667,
      "ece": 0.08080160000000003,
      "n": 375,
      "score": 83.83967999999999,
      "score_before_time_limit": 83.83967999999999,
      "time_limit_failed_share": 0.0
     }
    },
    "score": 56.87058116402116
   },
   "capability": 30.240174277642033,
   "composite_ci95": {
    "A": [
     0.007598161089799276,
     0.224928678737021
    ],
    "B": [
     0.004911677300807149,
     0.1551384520354836
    ],
    "C": [
     0.0052765007568050535,
     0.15620047134515344
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public bookable tariff. A 144.3M ModernBERT encoder with no decoder lineage, so the base-size-encoder line is used unchanged - USD 0.005 per 1M input and 0.0 per 1M output, the line the PUBLISHED v1.6 field already carries on modernbert-jev-v2, tacet-sonata and watt-flash-0.1 (verified in their own registry entries; modernbert-jev-v2 was measured in v1.6.8 on the earlier v1.6-regular-20261010-a2 draw, while tacet-sonata and watt-flash-0.1 are v1.6.1 rows - none of the three was measured on THIS draw, which carries no ModernBERT row), backed by the dated 25 Sep 2026 DeepInfra snapshot sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f in which thenlper/gte-base, BAAI/bge-base-en-v1.5, intfloat/e5-base-v2 and sentence-transformers/all-mpnet-base-v2 all price at USD 0.005 per 1M input and 0 output - and no new release-lane pricing line is created. Output tokens are a literal 0 reported by the author's own server (an encoder read-out; nothing is generated). x this row's own measured input tokens, which are the author's server's own count. Estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.001434 per 1,000 decisions (0.022x the official cost cap USD 0.064595), against USD 0.001440 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0014397966666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0014336531986531987
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Phocinae-Largha-150M-v1",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s into logs/CONDITIONS-<key>.jsonl - for this row's window 3.75 to 9.89, mean 5.81 over 12 samples, the quietest of the three new rows. The author's own phocinae-server 0.1.5 from PyPI, UNMODIFIED, run with their own documented line `PHOC_MODEL_DIR=<pinned snapshot> python -m phocinae.main` at PHOC_DEVICE=cpu, PHOC_THREADS=1 (their own documented CPU setting: 1.64 s per 5-decision case single-threaded), PHOC_EXTENSIONS=1 for their native option_scores, PHOC_PERM_AVG at its default 0 and no bearer token, inside a loopback-only network namespace with an outbound probe that failed as required and no credentials in the environment. Their package omits numpy from its declared dependencies although their own engine.py calls .numpy(); numpy 2.5.4 was installed so their code could run, and no file of theirs was edited. OUR shim (r107/src/phoc_shim_r107.py, byte-identical to the already source-reviewed r92/src/phoc_shim_r92.py, sha256 631cb60dde86b8e609382a32900db5e4ddc9fa9b2fef39bdb7b022dff558775e) is the only translation layer in front of it, because their published API carries no question text - see the mapping disclosure. Speed is a loaded-shared-CPU figure; the standard self-hosted x2 + 0.15 s adjustment is applied on top and the raw p50/p95 are kept. Re-measured for this release on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b) under CoS deputy decision #18209, at USD 0 on Sandy’s own CPU, 1 thread, 2026-10-13 05:59:36Z–06:11:07Z, 1-minute load average 3.75–9.89 (mean 5.81, 12 samples). DISCLOSED MAPPING: choice options are sent as \"<label>: <description>\" in the item’s own criteria order and the author’s option_scores are mapped back onto the bare labels by position; for noul only the author’s p(true) is forwarded and the stock adapter derives p(no) = 1 - p(yes), so their own rounded p(false) is unused. The author’s API answers score on a fixed 9-level scale 2..10 against items with 4 or 5 levels and forbids score options, so no identity mapping exists and this row uses pre-committed unsupported-native-score handling fixed before any measurement; all 375 score items of this draw are recorded as item-level misses (HTTP 422), and support.score is left at \"native\" rather than \"unsupported\" so that the type is still weighted and the misses count — the stricter of the two treatments, and the same rule every other row gets.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.178627926416878,
    "I_sealed": 3.04090685610893,
    "base": 3.609767391262904,
    "excess": -8.701805700047764,
    "gap": 1.137721070307948,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 12.535883779250636,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.7027027027027115,
       "hard": 5.630701478452351,
       "judge": 16.94133157547792,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100.0,
       "hard": -100.0,
       "judge": -100.0,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -163.63636363636363,
       "hard": -161.1111111111111,
       "judge": -213.55932203389827,
       "standard": -355.5555555555556
      }
     },
     "sealed|choice": {
      "cc": 9.122720568326791,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": -4.828462515883092,
       "hard": 9.448363594310154,
       "judge": 10.969637610186114,
       "standard": 12.676650495676029
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -100.0,
       "hard": -96.36363636363636,
       "judge": -93.84615384615384,
       "standard": -97.01492537313433
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -184.21052631578956,
       "hard": -207.887323943662,
       "judge": -209.375,
       "standard": -204.54545454545453
      }
     }
    }
   },
   "jevbench_score": 0.06552619942227712,
   "key": "phocinae-largha-150m-v1",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 (LICENSE + NOTICE in the weights repo; the author's phocinae-server is Apache-2.0 too)",
   "listing": "ranked",
   "measured_in": "v1.6.15",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Phocinae/Phocinae-Largha-150M-v1@7070e070fec3407061e76de42d7d87240ff5b74b",
   "noul_decisive": {
    "acc_among_decisive": 0.5454545454545454,
    "decisive": 11,
    "decisive_rate": 0.029333333333333333,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 56.87058116402116,
     "cost": 95.30667675976518,
     "intelligence": 3.609767391262904,
     "speed": 77.11764018052709
    },
    "board_release": "v1.6.43",
    "capability": 30.240174277642033,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 130,
    "capability_rank": 159,
    "composite_ci95": {
     "A": [
      0.007599423981329298,
      0.22503410752798958
     ],
     "B": [
      0.004912099447884301,
      0.15517856706924435
     ],
     "C": [
      0.005277377764812013,
      0.15627368578332607
     ]
    },
    "jevbench_score": 0.06554752053348605,
    "measured_in": "v1.6.15",
    "method": "v1.6",
    "open_board_rank": 181,
    "rank": 225,
    "ranks": {
     "A": 225,
     "B": 225,
     "C": 225,
     "capability": 228
    },
    "scores": {
     "A": 0.06554752053348605,
     "B": 0.04379268583036826,
     "C": 0.045519111481587535
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.0369279810023726,
     "p50_s_raw": 0.44346399050118634,
     "p95_s_adjusted": 1.8727826966009158,
     "p95_s_raw": 0.8613913483004579
    },
    "views": {
     "Balanced 33:33:33": 0.052037415444369725,
     "Emphasis on Accuracy 60:20:20": 0.030496990613082496,
     "Emphasis on Cost 20:20:60": 0.08106754873380384,
     "Emphasis on Speed 20:60:20": 0.07983832019244448,
     "Intelligence only": 0.018814714977998477
    }
   },
   "rank": 248,
   "ranked": true,
   "ranks": {
    "A": 248,
    "B": 248,
    "C": 248,
    "capability": 250
   },
   "repo": "https://huggingface.co/Phocinae/Phocinae-Largha-150M-v1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.06552619942227712,
    "B": 0.04378507107044704,
    "C": 0.04550430515435912
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.1184825160002219,
    "p50_s_raw": 0.484241258000111,
    "p95_s_adjusted": 1.9983378840101065,
    "p95_s_raw": 0.9241689420050533
   },
   "status": {
    "answered_ok": 1125,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.13636363636363635,
       "cc": -169.44444444444443,
       "cc_by_type": {
        "choice": -8.333333333333334,
        "noul": -100.0,
        "score": -400.0
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.22772277227722773,
       "cc": -92.00468363379287,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": -100.0,
        "score": -218.75
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.12871287128712872,
       "cc": -112.19743821369839,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -85.18518518518519,
        "score": -253.84615384615387
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.125,
       "cc": -78.57142857142856,
       "cc_by_type": {
        "choice": 14.285714285714292,
        "noul": -100.0,
        "score": -149.99999999999994
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0.0
      },
      "jaggedness": {
       "accuracy": 0.18888888888888888,
       "cc": -91.6056349618032,
       "cc_by_type": {
        "choice": 11.934552454282981,
        "noul": -94.5945945945946,
        "score": -192.156862745098
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.11182108626198083,
       "cc": -110.9060491326935,
       "cc_by_type": {
        "choice": 7.28185260191953,
        "noul": -100.0,
        "score": -240.0
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0.0
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -134.40860215053763,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.18518518518518517,
       "cc": -100.9327442317133,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100.0,
        "score": -228.57142857142856
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.13454545454545455,
       "cc": -96.41572945990254,
       "cc_by_type": {
        "choice": 5.11900880339095,
        "noul": -100.0,
        "score": -194.3661971830986
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.1511627906976744,
       "cc": -133.33333333333334,
       "cc_by_type": {
        "choice": 0.0,
        "noul": -80.0,
        "score": -320.0
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.25,
       "cc": 4.127829560585888,
       "cc_by_type": {
        "choice": 4.127829560585888
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 4.127829560585888
      },
      "perturb_pair": {
       "accuracy": 0.203125,
       "cc": -107.95410795410794,
       "cc_by_type": {
        "choice": 6.906906906906911,
        "noul": -100.0,
        "score": -230.76923076923072
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0.0
      },
      "usecase": {
       "accuracy": 0.14545454545454545,
       "cc": -83.8117647058823,
       "cc_by_type": {
        "choice": 11.764705882352976,
        "noul": -100.0,
        "score": -163.19999999999985
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -95.97069597069596,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -100.0,
        "score": -230.76923076923075
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.0,
       "cc": -63.15789473684211,
       "cc_by_type": {
        "choice": -26.315789473684216,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.15384615384615385,
       "cc": -107.64976958525347,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -100.0,
        "score": -240.0
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.125,
       "cc": -101.42362447095036,
       "cc_by_type": {
        "choice": 4.424778761061946,
        "noul": -100.0,
        "score": -208.69565217391303
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "el": {
       "accuracy": 0.0,
       "cc": -78.02607076350093,
       "cc_by_type": {
        "choice": -34.07821229050279,
        "noul": -100.0,
        "score": -100.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.158992180712424,
       "cc": -95.95231542756851,
       "cc_by_type": {
        "choice": 9.715527025542716,
        "noul": -95.68345323741008,
        "score": -201.88902007083814
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.1935483870967742,
       "cc": -140.48555800786872,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -100.0,
        "score": -352.94117647058823
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -66.12021857923497,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": -100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.07142857142857142,
       "cc": -98.45201238390091,
       "cc_by_type": {
        "choice": -5.882352941176468,
        "noul": -100.0,
        "score": -189.47368421052627
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.14285714285714285,
       "cc": -152.93296089385476,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100.0,
        "score": -375.0
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.2,
       "cc": -113.77352187952344,
       "cc_by_type": {
        "choice": 15.822291504286824,
        "noul": -100.0,
        "score": -257.1428571428571
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ja": {
       "accuracy": 0.04,
       "cc": -118.12549642573471,
       "cc_by_type": {
        "choice": -14.376489277204131,
        "noul": -100.0,
        "score": -240.0
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.0,
       "cc": -115.87301587301586,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -214.28571428571428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.21428571428571427,
       "cc": -61.11111111111111,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100.0,
        "score": -100.0
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "nl": {
       "accuracy": 0.1,
       "cc": -114.63203463203463,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -100.0,
        "score": -240.0
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.1794871794871795,
       "cc": -81.20540503301875,
       "cc_by_type": {
        "choice": 23.82564536606009,
        "noul": -100.0,
        "score": -167.44186046511635
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.10714285714285714,
       "cc": -102.78860227726348,
       "cc_by_type": {
        "choice": 4.537418974661164,
        "noul": -100.0,
        "score": -212.9032258064516
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -67.37588652482269,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100.0,
        "score": -100.0
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.15384615384615385,
       "cc": -103.98550724637681,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": -225.0
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.0,
       "cc": -93.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": -100.0,
        "score": -150.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -125.42372881355932,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.296,
       "cc": 9.068294456457268,
       "cc_by_type": {
        "choice": 9.068294456457268
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 9.068294456457268
      },
      "noul": {
       "accuracy": 0.016,
       "cc": -96.8,
       "cc_by_type": {
        "noul": -96.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.0,
       "cc": -203.0685920577614,
       "cc_by_type": {
        "score": -203.0685920577614
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.045454545454545456,
       "cc": -107.92156862745098,
       "cc_by_type": {
        "choice": -12.0,
        "noul": -100.0,
        "score": -211.76470588235296
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.125,
       "cc": -86.38318203535594,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -100.0,
        "score": -184.61538461538458
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.42857142857142855,
       "cc": -110.84010840108401,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -110.08771929824563,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": -225.0
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.17647058823529413,
       "cc": -91.48550724637681,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": -187.5
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.0,
       "cc": -341.9287211740041,
       "cc_by_type": {
        "choice": -25.786163522012576,
        "noul": -100.0,
        "score": -899.9999999999998
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.26666666666666666,
       "cc": -67.62192056309704,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -100.0,
        "score": -153.84615384615387
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -83.53279806089985,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100.0,
        "score": -156.52173913043475
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -72.22222222222223,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100.0,
        "score": -133.33333333333334
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.045454545454545456,
       "cc": -93.08935176172865,
       "cc_by_type": {
        "choice": -15.631691648822272,
        "noul": -100.0,
        "score": -163.63636363636368
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.12,
       "cc": -76.46229739252993,
       "cc_by_type": {
        "choice": 6.976744186046504,
        "noul": -100.0,
        "score": -136.36363636363632
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.1,
       "cc": -107.25806451612902,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100.0,
        "score": -225.0
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.1111111111111111,
       "cc": -91.82128385583282,
       "cc_by_type": {
        "choice": 7.869481765834927,
        "noul": -100.0,
        "score": -183.3333333333334
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.1152073732718894,
       "cc": -107.73538961038962,
       "cc_by_type": {
        "choice": 11.168831168831192,
        "noul": -100.0,
        "score": -234.37500000000006
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0.0
      },
      "moderation_trust_safety": {
       "accuracy": 0.16666666666666666,
       "cc": -89.5337021564864,
       "cc_by_type": {
        "choice": 7.869481765834927,
        "noul": -100.0,
        "score": -176.47058823529412
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.1111111111111111,
       "cc": -87.27233769016495,
       "cc_by_type": {
        "choice": -0.2785515320334252,
        "noul": -100.0,
        "score": -161.53846153846146
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.05555555555555555,
       "cc": -98.23239640683859,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100.0,
        "score": -176.47058823529412
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.043478260869565216,
       "cc": -91.98067632850244,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -100.0,
        "score": -160.00000000000006
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.125,
       "cc": -74.13373290878444,
       "cc_by_type": {
        "choice": 3.914590747330961,
        "noul": -100.0,
        "score": -126.31578947368425
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      51.501811957671954,
      59.20755983068783
     ],
     "I": [
      1.71972943365577,
      5.572698362718295
     ],
     "capability": [
      27.217574553966454,
      31.914331612070672
     ],
     "composite": {
      "A": [
       0.007598161089799276,
       0.224928678737021
      ],
      "B": [
       0.004911677300807149,
       0.1551384520354836
      ],
      "C": [
       0.0052765007568050535,
       0.15620047134515344
      ]
     },
     "seed": 57
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 56.87058116402116,
     "capability": 30.240174277642033,
     "intelligence": 3.609767391262904
    },
    "per_type": {
     "choice": {
      "accuracy": 0.296,
      "calibration": 86.7720634920635,
      "cc": 9.068294456457268,
      "cc_by_type": {
       "choice": 9.068294456457268
      },
      "cc_tiered": 10.829302173788713,
      "cc_tiered_open": 12.535883779250636,
      "cc_tiered_sealed": 9.122720568326791,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 9.068294456457268
     },
     "noul": {
      "accuracy": 0.016,
      "calibration": 83.83967999999999,
      "cc": -96.8,
      "cc_by_type": {
       "noul": -96.8
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.0,
      "calibration": null,
      "cc": -203.0685920577614,
      "cc_by_type": {
       "score": -203.0685920577614
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 53.88631212892281,
     "capability": 28.527882250386632,
     "intelligence": 3.169452371850454
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 1.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.05201949869554396,
    "Emphasis on Accuracy 60:20:20": 0.03049329752256628,
    "Emphasis on Cost 20:20:60": 0.08104145820610814,
    "Emphasis on Speed 20:60:20": 0.07976245200357382,
    "Intelligence only": 0.018814714977998477
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Kailune AI / @chi2liu (GitHub issue fstandhartinger/jevbench#145)",
   "axes": {
    "calibration": 84.15501405892886,
    "cost": 63.52461077797332,
    "intelligence": 28.75448619260912,
    "speed": 92.67163234372039
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07807774485134686,
      "ece_score": 84.38445102973063,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.38445102973063,
      "score_before_time_limit": 84.38445102973063,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.41018766756032177,
      "brier": 0.16186851482302253,
      "ece": 0.08062076189083324,
      "n": 373,
      "score": 83.87584762183336,
      "score_before_time_limit": 83.87584762183336,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9408602150537635,
      "interval90_width": 0.5562275985663083,
      "n": 372,
      "nrps": 0.12208071152631676,
      "score": 84.20474352522257,
      "score_before_time_limit": 84.20474352522257,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09691220898461597
     }
    },
    "score": 84.15501405892886
   },
   "capability": 56.45475012576899,
   "composite_ci95": {
    "A": [
     11.391972969610558,
     26.728013798311753
    ],
    "B": [
     9.375501093997054,
     23.291940599860233
    ],
    "C": [
     7.911092340007333,
     18.561120693272045
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference, carried over unchanged from PREREG-R36.md section 3.2. The UNMODIFIED frozen pricing_v15 path is asserted: openbmb/MiniCPM5-2B is in UNLISTED_BASE_MODELS - explicitly reviewed against both 25 Sep 2026 catalogs and not sold - so reference() returns no market reference rather than raising, and no exact base-model floor applies. The board already prices a row on THE SAME BASE this way: decision-2b (FlyMy.AI, openbmb/MiniCPM5-2B + LoRA and pointer head) carries the DeepInfra 2-4B size-class reference list price of USD 0.03 per 1M input and USD 0.0 per 1M output from the frozen 25 Sep 2026 snapshot (sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f). pjev-2b is the same shape: 2.52B, one forward pass with logits_to_keep=1, NOTHING GENERATED - the author's serve.py reports usage.output_tokens as a literal 0, not a decision count - so the output term is 0.0, not 0.15. Multiplied by this row's own measured input tokens. No new release-lane pricing line is created. Free weights are never cost 100 and never cost 0. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.016438 per 1,000 decisions (0.254x the official cost cap USD 0.064595), against USD 0.016274 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01627352
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01643789898989899
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "pjev-2b (Kailune AI, frozen MiniCPM5-2B + merged LoRA r64, letter-logit readout, T=1 stub)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment. The author's own serve.py / decision_core.py / letter_adapter.py, unmodified, at their declared GPU configuration: `python3 serve.py --model <pinned snapshot> --device cuda --host 127.0.0.1 --port <p>` at their own defaults (--max-tokens 16000), bfloat16 as hard-coded by the author, WITHOUT our earlier float32 patch of letter_adapter.py (PJEV_DTYPE; the flag is forbidden by the gate and absent from the launched argv/env). No deviation from the author's declared configuration other than the hardware. letter_adapter.dist() raises rather than truncating a prompt over the author's 16,000-token contract and serve.py maps that to HTTP 422, so the draw's 15 longest items are honest refusals scored as wrong (the same 15 items as in the CPU row). Speed is a dedicated-GPU figure, measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, torn down after the run (22 Oct 2026). The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven by three failed outbound probes before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.60865298275894,
    "I_sealed": 24.9003194024593,
    "base": 28.75448619260912,
    "excess": -2.13119319005607,
    "gap": 7.708333580299641,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 59.27670193309456,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 50.92796476879522,
       "judge": 52.537903757415954,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 11.725327812284332,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 13.043478260869556,
       "judge": 0.0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 26.823929202897954,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.07732888401458,
       "hard": 16.856096268424558,
       "judge": 33.35520857550455,
       "standard": 20.335976172376547
      }
     },
     "sealed|choice": {
      "cc": 52.21091228947275,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 50.04047646582629,
       "judge": 49.75514201762979,
       "standard": 52.5416578780848
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 10.344827586206895,
       "hard": -3.6363636363636376,
       "judge": -10.769230769230765,
       "standard": 7.462686567164178
      }
     },
     "sealed|score": {
      "cc": 22.49004591790515,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 20.380230279018463,
       "hard": 25.193643152818733,
       "judge": 19.747097400249913,
       "standard": 22.252182044004165
      }
     }
    }
   },
   "jevbench_score": 18.074372740962183,
   "key": "pjev-2b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 on the artifact (not gated, not private, not disabled). The three serving files are redistributed unmodified from the Eikos project (github.com/caiovicentino/eikos) under MIT, Copyright (c) 2026 Caio Vicentino; NOTICE and LICENSE-eikos are both present at the pinned revision.",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Kailune-AI/pjev-2b@ad60e59c7aa1f6a545c563bc404853f9a8067908",
   "noul_decisive": {
    "acc_among_decisive": 0.8772727272727273,
    "decisive": 220,
    "decisive_rate": 0.5898123324396782,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.466579728622,
     "cost": 63.52461077797332,
     "intelligence": 28.96851828460498,
     "speed": 49.10963929819891
    },
    "board_release": "v1.6.43",
    "capability": 57.71754900661349,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      10.021228315163317,
      23.41864571116037
     ],
     "B": [
      8.582473580841295,
      21.268305262838254
     ],
     "C": [
      6.959186329974525,
      16.26294841052804
     ]
    },
    "jevbench_score": 15.759681433648748,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": 118,
    "rank": 156,
    "ranks": {
     "A": 156,
     "B": 155,
     "C": 156,
     "capability": 145
    },
    "scores": {
     "A": 15.759681433648748,
     "B": 13.872903874213488,
     "C": 10.944223217811633
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 20.27664159500855,
     "p50_s_raw": 10.063320797504275,
     "p95_s_adjusted": 60.539669460893485,
     "p95_s_raw": 30.194834730446743
    },
    "views": {
     "Balanced 33:33:33": 13.755306662189126,
     "Emphasis on Accuracy 60:20:20": 11.592783816847163,
     "Emphasis on Cost 20:20:60": 15.856729233868942,
     "Emphasis on Speed 20:60:20": 14.54072118964529,
     "Intelligence only": 9.380636713101017
    }
   },
   "rank": 159,
   "ranked": true,
   "ranks": {
    "A": 159,
    "B": 166,
    "C": 164,
    "capability": 175
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/Kailune-AI/pjev-2b",
   "scores": {
    "A": 18.074372740962183,
    "B": 15.315756542769432,
    "C": 12.551647736779294
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22076407499994274,
    "p50_s_raw": 0.035382037499971375,
    "p95_s_adjusted": 0.2448546542999793,
    "p95_s_raw": 0.04742732714998965
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 47.192625784211685,
       "cc_by_type": {
        "choice": 41.666666666666664,
        "noul": 100.0,
        "score": -0.08878931403160095
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 47.192625784211685
      },
      "calibration_gen": {
       "accuracy": 0.45544554455445546,
       "cc": 2.779417852609811,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": -60.0,
        "score": 12.876959814457212
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 2.779417852609811
      },
      "composite_gen": {
       "accuracy": 0.5148514851485149,
       "cc": 14.586734299630178,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -25.92592592592593,
        "score": 18.46661662969451
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 14.586734299630178
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 69.23140098064862,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 47.69420294194586
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 69.23140098064862
      },
      "jaggedness": {
       "accuracy": 0.35,
       "cc": -13.833821090282077,
       "cc_by_type": {
        "choice": 29.258902791145346,
        "noul": -72.97297297297297,
        "score": 2.212606910981396
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6996805111821086,
       "cc": 48.30203158352211,
       "cc_by_type": {
        "choice": 61.96178568283877,
        "noul": 52.57731958762886,
        "score": 30.366989480098695
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 48.30203158352211
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -9.026636236663187,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50.0,
        "score": 9.518029434340336
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6072727272727273,
       "cc": 31.8492008743559,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": -2.777777777777779,
        "score": 36.47340778917284
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 31.8492008743559
      },
      "negation_pair_gen": {
       "accuracy": 0.5581395348837209,
       "cc": 14.453640352520686,
       "cc_by_type": {
        "choice": 56.66666666666668,
        "noul": -19.999999999999996,
        "score": 6.694254390895371
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 14.453640352520686
      },
      "option_order": {
       "accuracy": 0.5416666666666666,
       "cc": 41.41145139813582,
       "cc_by_type": {
        "choice": 41.41145139813582
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 41.41145139813582
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 64.47617964735888,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 80.0,
        "score": 46.46157197510967
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 64.47617964735888
      },
      "usecase": {
       "accuracy": 0.5681818181818182,
       "cc": 31.502274259633936,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 4.166666666666674,
        "score": 29.555842386744924
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 31.502274259633936
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 23.320769294368613,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50.0,
        "score": -3.8472159264179684
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 23.320769294368613
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 37.12677894367112,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 44.71367016434669
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 37.12677894367112
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 52.26829138448147,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 25.0,
        "score": 63.663281233090416
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 52.26829138448147
      },
      "el": {
       "accuracy": 0.4,
       "cc": 23.361075096743434,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -33.333333333333336,
        "score": 70.45566476881504
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 23.361075096743434
      },
      "en": {
       "accuracy": 0.5560382276281495,
       "cc": 24.818610987257575,
       "cc_by_type": {
        "choice": 51.45287462930642,
        "noul": 1.4388489208633004,
        "score": 21.564109411603006
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 24.818610987257575
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 52.511825617037566,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 75.0,
        "score": -17.464523148887313
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 52.511825617037566
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 11.324912795495031,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 65.66872745752335
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 11.324912795495031
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 47.62609454295699,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 64.05475421710626
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 47.62609454295699
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 8.604361095662563,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -50.0,
        "score": 9.332636359613389
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 8.604361095662563
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 57.17303362006391,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -0.4216629745705136
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 57.17303362006391
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 26.880921629481804,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 17.83100952704748
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 26.880921629481804
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 44.80451619025891,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0,
        "score": -21.142006984778817
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 44.80451619025891
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.90992448106408,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 85.3964401098589
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.90992448106408
      },
      "nl": {
       "accuracy": 0.4,
       "cc": 15.56120940445134,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -33.333333333333336,
        "score": 57.939039468765266
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 15.56120940445134
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 35.20767856477335,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 0.0,
        "score": 28.475342084502053
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 35.20767856477335
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 48.195957525907836,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 16.40873291950311
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 48.195957525907836
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": 1.7997955219120019,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -19.999999999999996,
        "score": 53.058961033821106
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 1.7997955219120019
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 31.137380604564715,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0.0,
        "score": 15.151272248476754
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 31.137380604564715
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.71888881665404,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 59.414730966091156
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.71888881665404
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 4.828693723155223,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -26.756856683641672
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 4.828693723155223
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.644,
       "cc": 54.01748981036759,
       "cc_by_type": {
        "choice": 54.01748981036759
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 54.01748981036759
      },
      "noul": {
       "accuracy": 0.5146666666666667,
       "cc": 2.9333333333333433,
       "cc_by_type": {
        "noul": 2.9333333333333433
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 2.9333333333333433
      },
      "score": {
       "accuracy": 0.4746666666666667,
       "cc": 23.713042630287838,
       "cc_by_type": {
        "score": 23.713042630287838
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 23.713042630287838
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": 6.415810838049524,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -50.0,
        "score": 17.247432514148574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.415810838049524
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 20.933052765336722,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 33.33333333333333,
        "score": -14.633553919310739
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 20.933052765336722
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 80.65848421901062,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 74.49577786028385
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 80.65848421901062
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -2.143072671679962,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": 8.775460347533215
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 24.66934193937999,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": 33.33333333333333,
        "score": 13.138460600748669
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 24.66934193937999
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -35.65918836172522,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -100.0,
        "score": -56.66309967637062
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 34.45878035079783,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -14.28571428571429,
        "score": 37.269898475362695
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.45878035079783
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": 0.7336197621111668,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50.0,
        "score": 25.37159099365057
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.7336197621111668
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 25.002590599297374,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60.0,
        "score": 55.84110513122545
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 25.002590599297374
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -0.6302270781537042,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 8.530446531420399
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 23.73256258574339,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": 11.111111111111116,
        "score": -5.029702423648397
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 23.73256258574339
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -5.525839680265041,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100.0,
        "score": 15.680545475333918
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 33.3680599381754,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -19.999999999999996,
        "score": 43.13680937306745
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 33.3680599381754
      },
      "model_routing": {
       "accuracy": 0.7419354838709677,
       "cc": 49.458148272289016,
       "cc_by_type": {
        "choice": 77.7922077922078,
        "noul": 54.54545454545454,
        "score": 16.036782479204703
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 49.458148272289016
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 28.4251098729428,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": 0.0,
        "score": 54.37321829445221
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 28.4251098729428
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 45.34747123856698,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 36.13526422637875
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 45.34747123856698
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 20.504013092940667,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 41.06868952512741
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 20.504013092940667
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 22.208565117404365,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0.0,
        "score": 24.596709844966714
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 22.208565117404365
      },
      "search_retrieval": {
       "accuracy": 0.7083333333333334,
       "cc": 55.98773712725733,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 50.0,
        "score": 49.99168113266167
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 55.98773712725733
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.72190811066586,
      85.45748339862506
     ],
     "I": [
      23.94121773861638,
      33.805531403151264
     ],
     "capability": [
      51.96793745679222,
      58.52381546627118
     ],
     "composite": {
      "A": [
       11.391972969610558,
       26.728013798311753
      ],
      "B": [
       9.375501093997054,
       23.291940599860233
      ],
      "C": [
       7.911092340007333,
       18.561120693272045
      ]
     },
     "seed": 58
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.15501405892886,
     "capability": 56.45475012576899,
     "intelligence": 28.75448619260912
    },
    "per_type": {
     "choice": {
      "accuracy": 0.644,
      "calibration": 84.38445102973063,
      "cc": 54.01748981036759,
      "cc_by_type": {
       "choice": 54.01748981036759
      },
      "cc_tiered": 55.74380711128366,
      "cc_tiered_open": 59.27670193309456,
      "cc_tiered_sealed": 52.21091228947275,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.01748981036759
     },
     "noul": {
      "accuracy": 0.5146666666666667,
      "calibration": 83.87584762183336,
      "cc": 2.9333333333333433,
      "cc_by_type": {
       "noul": 2.9333333333333433
      },
      "cc_tiered": 5.862663906142166,
      "cc_tiered_open": 11.725327812284332,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 2.9333333333333433
     },
     "score": {
      "accuracy": 0.4746666666666667,
      "calibration": 84.20474352522257,
      "cc": 23.713042630287838,
      "cc_by_type": {
       "score": 23.713042630287838
      },
      "cc_tiered": 24.656987560401554,
      "cc_tiered_open": 26.823929202897954,
      "cc_tiered_sealed": 22.49004591790515,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.713042630287838
     }
    },
    "raw_on_A": {
     "calibration": 79.07470646766168,
     "capability": 54.07122546865802,
     "intelligence": 29.067744469654354
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.18310508373799,
    "Emphasis on Accuracy 60:20:20": 12.636306415598074,
    "Emphasis on Cost 20:20:60": 17.820604749959447,
    "Emphasis on Speed 20:60:20": 19.94946868565402,
    "Intelligence only": 9.509919186674669
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Perplexity AI (perplexity-ai on Hugging Face)",
   "axes": {
    "calibration": 77.99716887881635,
    "cost": 64.42558238272366,
    "intelligence": 45.47596631214893,
    "speed": 91.46033110721706
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1388990410573218,
      "ece_score": 72.22019178853564,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 71.25725589802184,
      "score_before_time_limit": 72.22019178853564,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.21333333333333337,
      "brier": 0.1441461176011003,
      "ece": 0.09914083214795784,
      "n": 375,
      "score": 79.74425045803292,
      "score_before_time_limit": 80.17183357040844,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.8933333333333333,
      "interval90_width": 0.34911111111111115,
      "n": 375,
      "nrps": 0.10279383903889637,
      "score": 82.99000028039433,
      "score_before_time_limit": 83.65927447620396,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.11201033571851225
     }
    },
    "score": 77.99716887881635
   },
   "capability": 61.73656759548264,
   "composite_ci95": {
    "A": [
     38.693948316877766,
     67.76723917528828
    ],
    "B": [
     34.906459105171585,
     63.43735941092858
    ],
    "C": [
     26.870797442276217,
     47.63891253535204
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 6202ef78a2c83cbffc9389cf1f37df70fe0c85ed03f4dd5c5f997f3d31aae13a): declared base Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. USD per 1,000 decisions over the full 1,500-item denominator. Tokens as reported by the author's own `Decider.decide()` usage block (input = every non-padding position the request used; output 0, nothing is generated - the logits are read at the markers in one forward pass). The offline count through the author's own protocol.Encoder predicted the server's numbers EXACTLY, item by item: 1,939,205 input tokens over all 1,500 answered items, and the same prediction held on the 231 public items (154,452) before any sealed item was dispatched. COMMON COST BASIS: the preceding full-1,500-item figure is an all-items diagnostic. The published cost in this release uses the same remaining 1,485 items as every other row, under the unchanged v1.6.30 cost rule; USD 0.015339595959595961 per 1,000 decisions. All items still count for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0387841
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015339595959595961
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "pplx-decider-flash-v1.2 (Perplexity, Qwen3.5-4B full fine-tune, marker readout, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-19",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (compute capability 12.0, driver 580.178.04), FIN-02 (Finland, EU), pay-as-you-go non-spot, USD 2.395/h, instance id 0c5da6de-cf37-4754-ba2f-718e990499d4. The author's own `Decider.decide()` from the pinned commit, served behind our own 135-line stdlib HTTP shim (sha256 911747c9ed275c6f5734e4f0bf8466741600ef94960d80641139b1779822c349) on 127.0.0.1:8034 inside a network namespace with no route and a proven cut, offline and credential-free, one request at a time, reference path (fast=False, never fp8). Timed over loopback on the VM. Speed was measured while the host VM was downloading other weights; the measured server remained in its network namespace with the cut proven.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 50.15338750065794,
    "I_sealed": 40.79854512363991,
    "base": 45.47596631214893,
    "excess": -0.4846843933376803,
    "gap": 9.35484237701803,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 67.09257921129485,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 58.47750865051904,
       "judge": 68.35860250494397,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 43.55693581780538,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 39.81064747287361,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.609414282686686,
       "hard": 52.74275896247781,
       "judge": 37.96749037294228,
       "standard": 10.311776738655675
      }
     },
     "sealed|choice": {
      "cc": 58.79625662411246,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 52.642534983231165,
       "judge": 52.39960822722822,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 29.46198834618804,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 44.827586206896555,
       "hard": 23.636363636363633,
       "judge": 16.92307692307693,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 34.13739040061925,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 58.28165764661451,
       "hard": 31.500132808177707,
       "judge": 23.579999688936613,
       "standard": 43.17585803002867
      }
     }
    }
   },
   "jevbench_score": 54.0086854985917,
   "key": "pplx-decider-flash-v1.2",
   "last_measured_on": "2026-10-19",
   "licence": "Apache-2.0 (LICENSE + NOTICE in the repo and the HF metadata; base Qwen/Qwen3.5-4B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.31",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "perplexity-ai/pplx-decider-flash-v1.2@6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
   "noul_decisive": {
    "acc_among_decisive": 0.8703071672354948,
    "decisive": 293,
    "decisive_rate": 0.7855227882037533,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.68376661171601,
     "cost": 64.42558238272366,
     "intelligence": 46.343042310166375,
     "speed": 91.76093007933393
    },
    "board_release": "v1.6.43",
    "capability": 62.51340446094119,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 48,
    "capability_rank": 73,
    "composite_ci95": {
     "A": [
      41.08146712316482,
      68.44059689877392
     ],
     "B": [
      37.078763157794235,
      64.12134021639524
     ],
     "C": [
      28.5287966133089,
      49.54196301366422
     ]
    },
    "jevbench_score": 56.60442934796988,
    "measured_in": "v1.6.31",
    "method": "v1.6",
    "open_board_rank": 35,
    "rank": 59,
    "ranks": {
     "A": 59,
     "B": 61,
     "C": 65,
     "capability": 115
    },
    "scores": {
     "A": 56.60442934796988,
     "B": 52.200826013455654,
     "C": 39.30863149164575
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23424214299975574,
     "p50_s_raw": 0.04212107149987787,
     "p95_s_adjusted": 0.284604628999773,
     "p95_s_raw": 0.06730231449988651
    },
    "views": {
     "Balanced 33:33:33": 53.694309763093415,
     "Emphasis on Accuracy 60:20:20": 47.12180219293807,
     "Emphasis on Cost 20:20:60": 54.3430713515502,
     "Emphasis on Speed 20:60:20": 61.543638587494364,
     "Intelligence only": 39.81196500845765
    }
   },
   "rank": 68,
   "ranked": true,
   "ranks": {
    "A": 68,
    "B": 70,
    "C": 79,
    "capability": 140
   },
   "repo": "https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 54.0086854985917,
    "B": 49.67978278174221,
    "C": 37.506031596244235
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.238672914999961,
    "p50_s_raw": 0.0443364574999805,
    "p95_s_adjusted": 0.2993393091003099,
    "p95_s_raw": 0.07466965455015497
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 23.890753145000755,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100.0,
        "score": -86.66107389833108
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 23.890753145000755
      },
      "calibration_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 31.062896111775434,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": 19.999999999999996,
        "score": 33.63407539789257
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 31.062896111775434
      },
      "composite_gen": {
       "accuracy": 0.7821782178217822,
       "cc": 63.085637888757354,
       "cc_by_type": {
        "choice": 80.48780487804878,
        "noul": 55.55555555555556,
        "score": 53.2135532326677
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 63.085637888757354
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 73.17224732826755,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 59.516741984802614
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 73.17224732826755
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 27.336682346573767,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -2.7027027027026973,
        "score": 42.46058419863188
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 27.336682346573767
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 62.758982543003675,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 73.1958762886598,
        "score": 49.94943446441569
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.758982543003675
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -32.77757110043044,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75.0,
        "score": -24.363641136342864
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6327272727272727,
       "cc": 40.176758046887215,
       "cc_by_type": {
        "choice": 58.9175089664167,
        "noul": 22.222222222222232,
        "score": 39.39054295202272
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.176758046887215
      },
      "negation_pair_gen": {
       "accuracy": 0.7558139534883721,
       "cc": 47.13079509036586,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 33.33333333333333,
        "score": 14.725718604430915
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 47.13079509036586
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 86.6287203707147,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 71.89817312415613
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.6287203707147
      },
      "usecase": {
       "accuracy": 0.5454545454545454,
       "cc": 29.798882678319305,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": 8.333333333333325,
        "score": 30.082922544761825
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 29.798882678319305
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -1.3047979839876547,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50.0,
        "score": -58.67629871386772
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 22.06955126716424,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": -25.65002515396042
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 22.06955126716424
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 74.45606493117315,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100.0,
        "score": 63.191203643076975
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 74.45606493117315
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.042381528754813,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.8329173981825
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.042381528754813
      },
      "en": {
       "accuracy": 0.6715899218071243,
       "cc": 46.770940309835645,
       "cc_by_type": {
        "choice": 62.65605740715879,
        "noul": 38.84892086330935,
        "score": 38.80784265903879
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 46.770940309835645
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.42316226760575,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 9.05741502467694
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.42316226760575
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 28.01144305470872,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 49.06165156849775
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 28.01144305470872
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 43.274381438995476,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 72.17608549345701
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 43.274381438995476
      },
      "hi": {
       "accuracy": 0.42857142857142855,
       "cc": -19.93619014374724,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50.0,
        "score": -59.52924082230317
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 1.0,
       "cc": 85.83561523292173,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 57.50684569876518
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 85.83561523292173
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 23.825418841695438,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": 60.00000000000001,
        "score": 11.555684642639918
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 23.825418841695438
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -3.856451682651605,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 32.875089396489635
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.03423249272396,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.76936414483853
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.03423249272396
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -15.253284155506398,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 6.188199481532752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 53.750947209420126,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 56.48771255504834
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 53.750947209420126
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 65.2579740938948,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 48.80868951798375
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.2579740938948
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 24.622498041966832,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 30.463238806751548
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 24.622498041966832
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 40.810477316435026,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": -34.09030718112971
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 40.810477316435026
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 43.32203846875467,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": -5.517755561477933
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 43.32203846875467
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 7.235144418405379,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -19.537504597891207
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.235144418405379
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7026666666666667,
       "cc": 61.595131938996154,
       "cc_by_type": {
        "choice": 61.595131938996154
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.595131938996154
      },
      "noul": {
       "accuracy": 0.68,
       "cc": 36.00000000000001,
       "cc_by_type": {
        "noul": 36.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 36.00000000000001
      },
      "score": {
       "accuracy": 0.5653333333333334,
       "cc": 37.728820729894316,
       "cc_by_type": {
        "score": 37.728820729894316
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 37.728820729894316
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -32.3225310837819,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -75.0,
        "score": -41.96759325134569
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 4.685368541529724,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -22.520843995838714
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.685368541529724
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 51.559634286027205,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 53.45939066295966
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.559634286027205
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.66586536822346,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -16.12625107543062
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 12.927739876171662,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 30.08756745460194
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 12.927739876171662
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -83.98969555652734,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 0.0,
        "score": -276.49738855637446
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 32.191253901223796,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 1.8958905552119987
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 32.191253901223796
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 9.34672579166523,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50.0,
        "score": 30.30498573736502
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 9.34672579166523
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 3.1383334542369234,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60.0,
        "score": 52.7483336960441
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 3.1383334542369234
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 35.158166165270266,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 23.53302811750958
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 35.158166165270266
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 43.18587719402969,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 30.07442744772214
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.18587719402969
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -11.668195633176785,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -4.897060017809918
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 32.82004153950498,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 19.999999999999996,
        "score": 36.0416985148681
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 32.82004153950498
      },
      "model_routing": {
       "accuracy": 0.8064516129032258,
       "cc": 66.92352968172027,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 87.87878787878789,
        "score": 33.930762205334
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.92352968172027
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 9.844533432887033,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 52.56622985720236
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 9.844533432887033
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 47.70294210663885,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 43.20167683059438
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 47.70294210663885
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -5.190585022166457,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": -6.458454573888539
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 5.823091666909516,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 23.26637645000391
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 5.823091666909516
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 85.12622859667452,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 100.0,
        "score": 87.40715554091322
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.12622859667452
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      73.05211676751716,
      81.0760443951826
     ],
     "I": [
      39.610873870051066,
      50.26724676027738
     ],
     "capability": [
      57.416919513219,
      65.18283792776047
     ],
     "composite": {
      "A": [
       38.693948316877766,
       67.76723917528828
      ],
      "B": [
       34.906459105171585,
       63.43735941092858
      ],
      "C": [
       26.870797442276217,
       47.63891253535204
      ]
     },
     "seed": 59
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.99716887881635,
     "capability": 61.73656759548264,
     "intelligence": 45.47596631214893
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7026666666666667,
      "calibration": 71.25725589802184,
      "cc": 61.595131938996154,
      "cc_by_type": {
       "choice": 61.595131938996154
      },
      "cc_tiered": 62.94441791770365,
      "cc_tiered_open": 67.09257921129485,
      "cc_tiered_sealed": 58.79625662411246,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.595131938996154
     },
     "noul": {
      "accuracy": 0.68,
      "calibration": 79.74425045803292,
      "cc": 36.00000000000001,
      "cc_by_type": {
       "noul": 36.00000000000001
      },
      "cc_tiered": 36.50946208199671,
      "cc_tiered_open": 43.55693581780538,
      "cc_tiered_sealed": 29.46198834618804,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 36.00000000000001
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 82.99000028039433,
      "cc": 37.728820729894316,
      "cc_by_type": {
       "score": 37.728820729894316
      },
      "cc_tiered": 36.97401893674643,
      "cc_tiered_open": 39.81064747287361,
      "cc_tiered_sealed": 34.13739040061925,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 37.728820729894316
     }
    },
    "raw_on_A": {
     "calibration": 77.72244200758998,
     "capability": 62.348124101385025,
     "intelligence": 46.97380619518008
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 51.22654321341569,
    "Emphasis on Accuracy 60:20:20": 44.75146300467559,
    "Emphasis on Cost 20:20:60": 52.034156075505535,
    "Emphasis on Speed 20:60:20": 58.824894730166946,
    "Intelligence only": 37.61887464166974
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Perplexity AI (perplexity-ai on Hugging Face)",
   "axes": {
    "calibration": 89.96214607141948,
    "cost": 30.0417413123765,
    "intelligence": 71.71777888018025,
    "speed": 85.96624125269479
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04781048796115123,
      "ece_score": 90.43790240776976,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 89.23206370899949,
      "score_before_time_limit": 90.43790240776976,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.16266666666666663,
      "brier": 0.06680390854205732,
      "ece": 0.05588531133627415,
      "n": 375,
      "score": 88.34921539817053,
      "score_before_time_limit": 88.82293773274516,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.976,
      "interval90_width": 0.21555555555555558,
      "n": 375,
      "nrps": 0.045355161612738365,
      "score": 92.30515910708841,
      "score_before_time_limit": 93.04955555150042,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.046826863678626575
     }
    },
    "score": 89.96214607141948
   },
   "capability": 80.83996247579987,
   "composite_ci95": {
    "A": [
     20.27261362699533,
     20.828262097101554
    ],
    "B": [
     20.98539305622901,
     21.851574736643933
    ],
    "C": [
     20.27261362699533,
     20.828262097101554
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 6202ef78a2c83cbffc9389cf1f37df70fe0c85ed03f4dd5c5f997f3d31aae13a): declared base Qwen/Qwen3.8-27B -> qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output. Estimated, not charged, not a quote from the author or any host. USD per 1,000 decisions on the denominator of the release basis (the same-items cost rule for this draw prices the common item set without the 15 very long items, for every row alike). Same pair as the published pplx-decider-v1-1-27b and vllm-sr-d3 rows. Tokens as reported by the author's own `Decider.decide()` usage block (input = every non-padding position the request used; output 0, nothing is generated - the logits are read at the markers in one forward pass). The offline count through the author's own protocol.Encoder predicted the server's numbers EXACTLY, item by item: 1,939,205 input tokens over all 1,500 answered items, and the same prediction held on the 231 public items (154,452) before any sealed item was dispatched. OVER THE OFFICIAL COST CAP on the common 1,485-item basis: USD 0.214754343 per 1,000 decisions (3.325x cap); the row stays RANKED, but is outside the official cost-capped views and visible in uncapped views. ALL-ITEMS DIAGNOSTIC: USD 0.542977 per 1,000 over all 1,500 answered items (8.41x cap). All 1,500 items count for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5429773999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.2147543434343434
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "pplx-decider-v1.2 (Perplexity, Qwen3.8-27B full fine-tune, marker readout, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-20",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA H200 141 GB (compute capability 9.0, driver 580.178.04), FIN-02 (Finland, EU), pay-as-you-go non-spot, instance type 2H200.141S.88V at USD 11.32/h with the second GPU unbound from the driver before anything ran (nvidia-smi and torch both saw exactly one GPU; single-GPU, no tensor parallelism). The author's own `Decider.decide()` from the pinned commit, served behind our own 135-line stdlib HTTP shim (sha256 911747c9ed275c6f5734e4f0bf8466741600ef94960d80641139b1779822c349) on 127.0.0.1:8035 inside a network namespace with no route and a proven cut, offline and credential-free, one request at a time, reference path (fast=False, never fp8). Timed over loopback on the VM. Measured 20 Oct 2026.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 77.10920374861246,
    "I_sealed": 66.32635401174804,
    "base": 71.71777888018025,
    "excess": 0.9433229665087097,
    "gap": 10.78284973686442,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 86.74318374569839,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 96.044825313118,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 73.42650103519671,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 91.30434782608697,
       "judge": 50.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 71.15792646494232,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.33980083493176,
       "hard": 77.46041640319304,
       "judge": 72.97602843031399,
       "standard": 53.734856455388666
      }
     },
     "sealed|choice": {
      "cc": 80.44759269360293,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 74.49982652943217,
       "judge": 81.48873653281098,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 54.272083037909084,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 45.45454545454546,
       "judge": 50.76923076923077,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 64.25938630373211,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.77887937684766,
       "hard": 61.98686138973364,
       "judge": 57.77258889166228,
       "standard": 68.27488571327603
      }
     }
    }
   },
   "jevbench_score": 20.634956555436453,
   "key": "pplx-decider-v1.2",
   "last_measured_on": "2026-10-20",
   "licence": "Apache-2.0 (LICENSE + NOTICE in the repo and the HF metadata; base Qwen/Qwen3.8-27B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.33",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "perplexity-ai/pplx-decider-v1.2@225c913ddc8761a8264f04f64548e1f97e443eb3",
   "noul_decisive": {
    "acc_among_decisive": 0.9648562300319489,
    "decisive": 313,
    "decisive_rate": 0.839142091152815,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.77013189733844,
     "cost": 30.0417413123765,
     "intelligence": 72.71358625871065,
     "speed": 86.32357007699105
    },
    "board_release": "v1.6.43",
    "capability": 81.74185907802455,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.385257230642424,
      20.922079167304812
     ],
     "B": [
      21.116487641783483,
      21.96657076729541
     ],
     "C": [
      20.385257230642424,
      20.922079167304812
     ]
    },
    "jevbench_score": 20.73512390064423,
    "measured_in": "v1.6.33",
    "method": "v1.6",
    "open_board_rank": 91,
    "rank": 125,
    "ranks": {
     "A": 125,
     "B": 118,
     "C": 107,
     "capability": 17
    },
    "scores": {
     "A": 20.73512390064423,
     "B": 21.644559304900415,
     "C": 20.73512390064423
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.43327262599992766,
     "p50_s_raw": 0.14163631299996382,
     "p95_s_adjusted": 0.5381232308001244,
     "p95_s_raw": 0.1940616154000622
    },
    "views": {
     "Balanced 33:33:33": 18.473812614761474,
     "Emphasis on Accuracy 60:20:20": 20.9570557390737,
     "Emphasis on Cost 20:20:60": 14.41726658567706,
     "Emphasis on Speed 20:60:20": 22.068170846951432,
     "Intelligence only": 26.249785582202126
    }
   },
   "rank": 142,
   "ranked": true,
   "ranks": {
    "A": 142,
    "B": 136,
    "C": 124,
    "capability": 27
   },
   "repo": "https://huggingface.co/perplexity-ai/pplx-decider-v1.2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.634956555436453,
    "B": 21.508121467149618,
    "C": 20.634956555436453
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.44363213599992835,
    "p50_s_raw": 0.14681606799996416,
    "p95_s_adjusted": 0.5706277350999812,
    "p95_s_raw": 0.2103138675499906
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 73.07521976445251,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 27.55899262669086
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 73.07521976445251
      },
      "calibration_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 57.398871595668176,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": 52.0,
        "score": 64.73532104363233
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 57.398871595668176
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 86.73438142756389,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 85.18518518518519,
        "score": 78.26999161783166
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 86.73438142756389
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 84.4704120616969,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 93.41123618509071
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.4704120616969
      },
      "jaggedness": {
       "accuracy": 0.7444444444444445,
       "cc": 59.604085228758656,
       "cc_by_type": {
        "choice": 65.35129932627527,
        "noul": 35.13513513513513,
        "score": 78.32582122486556
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 59.604085228758656
      },
      "llm_router": {
       "accuracy": 0.9361022364217252,
       "cc": 85.56210896843349,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 95.8762886597938,
        "score": 65.56481503515184
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 85.56210896843349
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 20.88632537167448,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -25.0,
        "score": 49.51464621811623
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 20.88632537167448
      },
      "multilingual": {
       "accuracy": 0.7781818181818182,
       "cc": 62.04880430430405,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 38.888888888888886,
        "score": 65.84246044397761
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 62.04880430430405
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 80.71787667169285,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 73.33333333333334,
        "score": 68.82029668174519
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 80.71787667169285
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 79.18647345233632,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 52.574435372023984
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.18647345233632
      },
      "usecase": {
       "accuracy": 0.75,
       "cc": 59.529698223943285,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 37.5,
        "score": 64.37340839732005
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.529698223943285
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 48.013941666516395,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": 32.13706309478728
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 48.013941666516395
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 53.588644483199374,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 27.432600116264794
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 53.588644483199374
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.0741245994998,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 85.11617910823391
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.0741245994998
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 61.07592115708842,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0.0,
        "score": 99.9875400075781
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 61.07592115708842
      },
      "en": {
       "accuracy": 0.8349261511728931,
       "cc": 72.82231693818183,
       "cc_by_type": {
        "choice": 82.64605020685615,
        "noul": 66.90647482014388,
        "score": 68.91442578754545
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 72.82231693818183
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 77.50244222824413,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 42.295254906592106
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 77.50244222824413
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.96563885246287,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 97.35046846995695
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.96563885246287
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 90.44568219340927,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 92.51351716846314
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 90.44568219340927
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 42.92298599748088,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25.0,
        "score": 37.28851106506833
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 42.92298599748088
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 73.01823353533898,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 33.08431868863578
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.01823353533898
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 34.74270854767963,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 0.0,
        "score": 18.525186802689397
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 34.74270854767963
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 20.91054015944991,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 62.73162047834971
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 20.91054015944991
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.64112914545721,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.5900541030383
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.64112914545721
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -25.467887068329684,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": -50.429635230963086
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 77.98547426709224,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 69.19129372806471
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 77.98547426709224
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 77.04784638664607,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 62.964399501717814
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 77.04784638664607
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 53.66153195458858,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 52.04842565099979
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 53.66153195458858
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 84.37759555821967,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 53.132786674659
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 84.37759555821967
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.70355972828668,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 95.36874370098906
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.70355972828668
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 49.13494079901958,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 14.071489063725428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 49.13494079901958
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8613333333333333,
       "cc": 82.08920951415067,
       "cc_by_type": {
        "choice": 82.08920951415067
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.08920951415067
      },
      "noul": {
       "accuracy": 0.8053333333333333,
       "cc": 61.06666666666667,
       "cc_by_type": {
        "noul": 61.06666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 61.06666666666667
      },
      "score": {
       "accuracy": 0.7546666666666667,
       "cc": 67.14016711842746,
       "cc_by_type": {
        "score": 67.14016711842746
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.14016711842746
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 10.59424018776719,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -50.0,
        "score": 13.782720563301565
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 10.59424018776719
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 87.58326240675524,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 62.74978722026573
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 87.58326240675524
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 38.482668211129294,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 14.228492438265938
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 38.482668211129294
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -10.890476546823086,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -95.82932437731135
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 38.4736130134271,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 77.73967961999143
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 38.4736130134271
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 33.292688730256714,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": -20.876650790361918
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.292688730256714
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 38.126345773148394,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 58.076516311041814
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 38.126345773148394
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 60.05026660124205,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 80.15079980372617
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 60.05026660124205
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 15.085726936490197,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60.0,
        "score": 46.92384747613726
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 15.085726936490197
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 74.46899896013086,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 69.58829595248395
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 74.46899896013086
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 63.797278280314096,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 47.464186262130944
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.797278280314096
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": 11.921579031604551,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 31.463661825996457
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.921579031604551
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 59.44426920442447,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 69.84912239254406
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.44426920442447
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 82.17204853771132,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 54.221773318761656
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.17204853771132
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 18.622337771983307,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 78.89964287449116
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 18.622337771983307
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 56.74843643361415,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 70.33815981152026
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.74843643361415
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 31.118849344180674,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": 22.913198278847435
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 31.118849344180674
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 27.996149618548205,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 75.29279668173157
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 27.996149618548205
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.85528914958637,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.56586744875911
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.85528914958637
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.78487270534086,
      91.02037761468912
     ],
     "I": [
      67.3887930401396,
      75.29632276772642
     ],
     "capability": [
      77.17563485429892,
      82.6025660243471
     ],
     "composite": {
      "A": [
       20.27261362699533,
       20.828262097101554
      ],
      "B": [
       20.98539305622901,
       21.851574736643933
      ],
      "C": [
       20.27261362699533,
       20.828262097101554
      ]
     },
     "seed": 60
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.96214607141948,
     "capability": 80.83996247579987,
     "intelligence": 71.71777888018025
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8613333333333333,
      "calibration": 89.23206370899949,
      "cc": 82.08920951415067,
      "cc_by_type": {
       "choice": 82.08920951415067
      },
      "cc_tiered": 83.59538821965066,
      "cc_tiered_open": 86.74318374569839,
      "cc_tiered_sealed": 80.44759269360293,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.08920951415067
     },
     "noul": {
      "accuracy": 0.8053333333333333,
      "calibration": 88.34921539817053,
      "cc": 61.06666666666667,
      "cc_by_type": {
       "noul": 61.06666666666667
      },
      "cc_tiered": 63.849292036552896,
      "cc_tiered_open": 73.42650103519671,
      "cc_tiered_sealed": 54.272083037909084,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 61.06666666666667
     },
     "score": {
      "accuracy": 0.7546666666666667,
      "calibration": 92.30515910708841,
      "cc": 67.14016711842746,
      "cc_by_type": {
       "score": 67.14016711842746
      },
      "cc_tiered": 67.70865638433722,
      "cc_tiered_open": 71.15792646494232,
      "cc_tiered_sealed": 64.25938630373211,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.14016711842746
     }
    },
    "raw_on_A": {
     "calibration": 88.31082903662588,
     "capability": 79.78253309681389,
     "intelligence": 71.25423715700191
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.398770171284372,
    "Emphasis on Accuracy 60:20:20": 20.80703022031299,
    "Emphasis on Cost 20:20:60": 14.389784473577171,
    "Emphasis on Speed 20:60:20": 21.9780444481153,
    "Intelligence only": 25.890296640553807
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Olaverse (Hugging Face olaverse; benchmarkheaven.com/submit ref 6195A889)",
   "axes": {
    "calibration": 83.64479179791499,
    "cost": null,
    "intelligence": 36.57669575617585,
    "speed": 87.71257539670552
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08471827409306511,
      "ece_score": 83.05634518138699,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 83.05634518138699,
      "score_before_time_limit": 83.05634518138699,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3699731903485255,
      "brier": 0.14497102452798155,
      "ece": 0.08867916523731106,
      "n": 373,
      "score": 82.26416695253779,
      "score_before_time_limit": 82.26416695253779,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9247311827956989,
      "interval90_width": 0.39471326164874576,
      "n": 372,
      "nrps": 0.10271997153867293,
      "score": 85.61386325982022,
      "score_before_time_limit": 85.61386325982022,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09250138163246135
     }
    },
    "score": 83.64479179791499
   },
   "capability": 60.11074377704542,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 6202ef78..., the module in force at scoring time; the later re-freezes only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base (the module RAISES 'base model lacks an explicit market-listing decision', for every spelling tried); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void - the same treatment as the live jevora-9b-v1 and kev-9b-v2 rows on that same declared base. The tokens needed for a later reprice are measured and on file: the author's server reports NO usage block at all, so the input tokens were counted OFFLINE with the author's own pinned tokenizer and their own unchanged encode() on the exact prompts their server builds from the runner's request bodies (CoS deputy decision #18974, as accepted for the sibling row): 700,492 input tokens over the 1,485 answered items, 0 output (nothing is generated). That count is byte-for-byte identical, item by item, to the count accepted for purplemist-mini-1.0 - same rendered-prompt-ids sha256 ad0810696b7e80eba87192caa70f57c19cdbee103e37f3d9cd163ef5df434c86 - which it must be, because the rendered prompt depends only on the tokenizer, encode(), the two flags and the request body, all byte-identical between the two rows, and not on the weights.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "PurpleMIST-Flash-1.0 (Olaverse)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-17",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod, 1x NVIDIA RTX 6000 Ada Generation 48 GB (sm_89, driver 595.84), Athens, Greece (EU), secure tier (not preemptible), USD 0.80/h, 05:26-05:53Z on 17 Oct 2026 = USD 0.35; removed by exact id and verified gone four ways. Speed is as measured on that card and is NOT strictly comparable to the H100/RTX 5090 rows. Isolation: the author's server ran inside a network namespace with loopback only, with the cut PROVEN before any sealed item was dispatched (no interfaces but lo, no routes, outbound by name and by IP both failing) and proven again immediately before the first sealed request; credential-free environment (env -i, checked), weights from a locally verified offline snapshot of the pinned revision (11 files, 15,927,059,974 bytes, model.safetensors sha256 2dba83efd90dbe2fcf767eaa004f48789029cff3282f7c6ae8a66eca7dc26202), sealed input deleted on the pod at 05:51:55Z and the server's process group killed by exact pid. One request at a time. Transfer deviation: the Hugging Face Xet path stalled on this host (its own log reported 'connection struggling' at a success ratio of 1.000 while the link was taking 25 MB/s), so the snapshot was fetched over plain HTTPS with HF_HUB_DISABLE_XET=1; what is verified did not change.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.963930901767135,
    "I_sealed": 31.18946061058456,
    "base": 36.57669575617585,
    "excess": 0.934943520826863,
    "gap": 10.774470291182574,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 63.66615272089087,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 60.36489462094998,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 17.12008281573499,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 45.105557168675546,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.05582660274707,
       "hard": 49.94314086701773,
       "judge": 35.26365964380409,
       "standard": 45.718101342262564
      }
     },
     "sealed|choice": {
      "cc": 57.693838803821905,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 48.47924135538336,
       "judge": 56.80705190989228,
       "standard": 65.82999367222105
      }
     },
     "sealed|noul": {
      "cc": 4.470946449330393,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 31.034482758620683,
       "hard": -3.6363636363636376,
       "judge": -1.538461538461533,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 31.403596578601384,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 61.046601730525076,
       "hard": 25.44469977308672,
       "judge": 23.561584549581017,
       "standard": 40.2629056571994
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "purplemist-flash-1.0",
   "last_measured_on": "2026-10-17",
   "licence": "Apache-2.0 (model card and HF metadata), public, not gated",
   "listing": "listed",
   "measured_in": "v1.6.28",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "olaverse/PurpleMIST-Flash-1.0@a48fe0206ee0d8a3f4458a3581468c24a7365ac5",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.8851063829787233,
    "decisive": 235,
    "decisive_rate": 0.6300268096514745,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.64479179791499,
     "cost": null,
     "intelligence": 36.57669575617585,
     "speed": 88.25066979413353
    },
    "board_release": "v1.6.43",
    "capability": 60.11074377704542,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.28",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.31827442100111514,
     "p50_s_raw": 0.08413721050055756,
     "p95_s_adjusted": 0.4700361753995821,
     "p95_s_raw": 0.16001808769979106
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/olaverse/PurpleMIST-Flash-1.0",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.33086025799857455,
    "p50_s_raw": 0.09043012899928726,
    "p95_s_adjusted": 0.511797141898569,
    "p95_s_raw": 0.18089857094928447
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 70.39546327561969,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 44.51972316019237
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 70.39546327561969
      },
      "calibration_gen": {
       "accuracy": 0.49504950495049505,
       "cc": 17.209450111800084,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": -12.0,
        "score": 20.892401236778824
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 17.209450111800084
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 35.56560256341613,
       "cc_by_type": {
        "choice": 57.72357723577236,
        "noul": 18.518518518518512,
        "score": 30.454711935957533
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 35.56560256341613
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 72.27956011126733,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 56.838680333801975
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 72.27956011126733
      },
      "jaggedness": {
       "accuracy": 0.46111111111111114,
       "cc": 2.3307448578111356,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -56.75675675675676,
        "score": 21.496825786398045
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 2.3307448578111356
      },
      "llm_router": {
       "accuracy": 0.7380191693290735,
       "cc": 56.233400214405066,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 46.39175257731958,
        "score": 51.62957160204073
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.233400214405066
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -17.44240188691204,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75.0,
        "score": -3.1004015370247773
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6363636363636364,
       "cc": 38.55335313614513,
       "cc_by_type": {
        "choice": 62.83012716009129,
        "noul": 2.777777777777768,
        "score": 50.052154470566315
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 38.55335313614513
      },
      "negation_pair_gen": {
       "accuracy": 0.5348837209302325,
       "cc": 15.35444594072437,
       "cc_by_type": {
        "choice": 53.333333333333336,
        "noul": -19.999999999999996,
        "score": 12.730004488839764
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 15.35444594072437
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 78.93817096043851,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100.0,
        "score": 72.85054891735155
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.93817096043851
      },
      "usecase": {
       "accuracy": 0.5863636363636363,
       "cc": 32.92209243846133,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": 8.333333333333325,
        "score": 34.550591040874146
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 32.92209243846133
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -1.182611711477912,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0.0,
        "score": -27.357358943957543
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 52.0471958671179,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 22.808254268020377
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.0471958671179
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 77.26496186483496,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 80.68869090423941
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.26496186483496
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 16.45188091170487,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 99.82119208334552
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.45188091170487
      },
      "en": {
       "accuracy": 0.6168549087749783,
       "cc": 34.58332742333039,
       "cc_by_type": {
        "choice": 58.48232264678242,
        "noul": 11.51079136690647,
        "score": 33.75686825630226
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 34.58332742333039
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 57.29713259637261,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75.0,
        "score": 6.679326010977515
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 57.29713259637261
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 29.410935130872797,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 87.13990921775499
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 29.410935130872797
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 53.565431529597426,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 81.87276517702757
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 53.565431529597426
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 13.66047649462213,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25.0,
        "score": -0.4990174435079098
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 13.66047649462213
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 56.84148427361694,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -1.416311013911442
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 56.84148427361694
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 26.796685072403772,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 17.578299855813384
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 26.796685072403772
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -5.669662434756812,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 27.435457140174012
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.54577863099985,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.63733589299954
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.54577863099985
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 7.617863381918501,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 74.80164209380746
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.617863381918501
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 47.5434181308745,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 37.86512531941148
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 47.5434181308745
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 33.36042327927317,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": -19.999999999999996,
        "score": 51.902130179599126
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 33.36042327927317
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 11.505633172783869,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 56.64455909281969
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 11.505633172783869
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 0.08992728763732316,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -50.0,
        "score": -6.251957267522812
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.08992728763732316
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.95162904215476,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 92.37101615872234
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.95162904215476
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 19.93758488161902,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -73.52057868847626
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.93758488161902
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.688,
       "cc": 59.70072140683901,
       "cc_by_type": {
        "choice": 59.70072140683901
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.70072140683901
      },
      "noul": {
       "accuracy": 0.5546666666666666,
       "cc": 10.933333333333328,
       "cc_by_type": {
        "noul": 10.933333333333328
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 10.933333333333328
      },
      "score": {
       "accuracy": 0.5573333333333333,
       "cc": 35.98058722164548,
       "cc_by_type": {
        "score": 35.98058722164548
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.98058722164548
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -8.023171759301837,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -25.0,
        "score": -51.06951527790551
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 18.05749693437015,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 33.33333333333333,
        "score": -4.6266810395396485
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 18.05749693437015
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 28.239231994773096,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 33.49818378919734
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 28.239231994773096
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -13.419745719245478,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": 29.916201438754797
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 19.48037661141311,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 49.74547766032629
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 19.48037661141311
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -47.63928647397137,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100.0,
        "score": -54.8675449565053
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 40.90857018259701,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 28.047839399331654
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 40.90857018259701
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -9.471877723757162,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100.0,
        "score": 34.30213686357171
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 34.06036190912817,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 63.847752394051184
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.06036190912817
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 16.292238984459793,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0.0,
        "score": 38.81247712468559
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 16.292238984459793
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 38.280305827415546,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 37.57993557010195
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 38.280305827415546
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -32.35528164753511,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100.0,
        "score": -0.2916513942182286
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 35.00919149670052,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 19.999999999999996,
        "score": 19.576518827913468
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 35.00919149670052
      },
      "model_routing": {
       "accuracy": 0.783410138248848,
       "cc": 58.2495546137415,
       "cc_by_type": {
        "choice": 87.14285714285714,
        "noul": 51.515151515151516,
        "score": 36.090655183215844
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 58.2495546137415
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 3.368242149671545,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": 67.68630034536781
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.368242149671545
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 44.13000314873262,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 33.33333333333333,
        "score": 49.19595187888124
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 44.13000314873262
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 21.425366213581725,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -50.0,
        "score": 14.276098640745172
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 21.425366213581725
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 15.648163302975552,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 52.74159135820202
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 15.648163302975552
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 85.73439203830024,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100.0,
        "score": 78.55548928216051
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.73439203830024
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.09728138937768,
      85.59062383959186
     ],
     "I": [
      31.13234243642542,
      41.14289724655533
     ],
     "capability": [
      55.33910988485174,
      62.724058487049085
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 61
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.64479179791499,
     "capability": 60.11074377704542,
     "intelligence": 36.57669575617585
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 83.05634518138699,
      "cc": 59.70072140683901,
      "cc_by_type": {
       "choice": 59.70072140683901
      },
      "cc_tiered": 60.679995762356384,
      "cc_tiered_open": 63.66615272089087,
      "cc_tiered_sealed": 57.693838803821905,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.70072140683901
     },
     "noul": {
      "accuracy": 0.5546666666666666,
      "calibration": 82.26416695253779,
      "cc": 10.933333333333328,
      "cc_by_type": {
       "noul": 10.933333333333328
      },
      "cc_tiered": 10.79551463253269,
      "cc_tiered_open": 17.12008281573499,
      "cc_tiered_sealed": 4.470946449330393,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 10.933333333333328
     },
     "score": {
      "accuracy": 0.5573333333333333,
      "calibration": 85.61386325982022,
      "cc": 35.98058722164548,
      "cc_by_type": {
       "score": 35.98058722164548
      },
      "cc_tiered": 38.25457687363846,
      "cc_tiered_open": 45.105557168675546,
      "cc_tiered_sealed": 31.403596578601384,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.98058722164548
     }
    },
    "raw_on_A": {
     "calibration": 86.19021939188609,
     "capability": 62.4812521197941,
     "intelligence": 38.77228484770211
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Olaverse (contact@olaverse.co.uk)",
   "axes": {
    "calibration": 77.68611335416006,
    "cost": 70.75879834655666,
    "intelligence": 8.377579633895749,
    "speed": 88.54013979409211
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1300396062800142,
      "ece_score": 73.99207874399715,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 73.99207874399715,
      "score_before_time_limit": 73.99207874399715,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.9142091152815013,
      "brier": 0.22633030087611059,
      "ece": 0.12555655399612065,
      "n": 373,
      "score": 74.88868920077587,
      "score_before_time_limit": 74.88868920077587,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9354838709677419,
      "interval90_width": 0.7486559139784947,
      "n": 372,
      "nrps": 0.16595993804336154,
      "score": 84.17757211770714,
      "score_before_time_limit": 84.17757211770714,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07524430980124781
     }
    },
    "score": 77.68611335416006
   },
   "capability": 43.0318464940279,
   "composite_ci95": {
    "A": [
     0.2325661652541505,
     1.61193012940895
    ],
    "B": [
     0.15962297864662708,
     1.1885050789163212
    ],
    "C": [
     0.16150428142649342,
     1.1193959232006596
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference via the frozen module's Base-to-listed-model mapping (Qwen/Qwen3.5-2B-Base -> deepinfra:Qwen/Qwen3.5-2B), from the frozen 25 Sep 2026 snapshot through the UNMODIFIED pricing_v15 path (sha256 dae86740...): declared base Qwen/Qwen3.5-2B-Base -> deepinfra:Qwen/Qwen3.5-2B USD 0.02/M input, USD 0.1/M output. The author's server reports NO token usage, so the input tokens were counted offline with the author's own pinned tokenizer and their own unchanged encode() on the exact prompts their server builds from the runner's request bodies (CoS deputy decision #18974; receipt r109/receipts/TOKCOUNT-PURPLEMIST-R109.json; independent recount PASS, r109/review-tokcount/REVIEW.json): 700,492 input tokens over the 1,485 answered items. Nothing is generated (one forward pass, pointer-head option probabilities), so output is 0 under the 9 Oct missing-output rule. Cost on the common 1,485-item basis. Estimated, not a bill. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.009434 per 1,000 decisions (0.146x the official cost cap USD 0.064595), against USD 0.009340 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009339893333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00943423569023569
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "PurpleMIST-Mini-1.0 (Olaverse)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment. The author's own serve.py and purplemist.py, unchanged, at their declared GPU configuration: `python3 serve.py --model <pinned snapshot> --device cuda --max-len 65536 --no-truncate --host 127.0.0.1 --port <p>` (fp32 loaded automatically as the model's inference_dtype), with the author's server WARM-UP ON (the earlier CPU run used our --no-warmup, which is forbidden by the gate and absent from the launched argv). No deviation from the author's declared configuration other than the hardware. With --no-truncate the author's code refuses rather than truncates a prompt over 65,536 tokens, so the draw's 15 longest items are honest HTTP 422 refusals scored as wrong (the same 15 as in the CPU row). The author's server reports no token usage, so input tokens were counted offline with the author's pinned tokenizer (700,492 over the 1,485 answered items; same count as the CPU row). Speed is a dedicated-GPU figure, measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, torn down after the run (22 Oct 2026). The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven by three failed outbound probes before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 10.02427738947358,
    "I_sealed": 6.730881878317918,
    "base": 8.377579633895749,
    "excess": -6.546131259200049,
    "gap": 3.293395511155662,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 21.510322262497397,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.94594594594595,
       "hard": 1.8559295375904445,
       "judge": 32.76203032300594,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -91.30434782608697,
       "judge": -100.0,
       "standard": -100.0
      }
     },
     "open|score": {
      "cc": 8.562509905923346,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.11955505993331,
       "hard": -13.404645221049183,
       "judge": 24.543976136562172,
       "standard": 4.746098236905183
      }
     },
     "sealed|choice": {
      "cc": 20.192645634953756,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 19.94917407878019,
       "hard": 17.774950850005773,
       "judge": 21.547502448579845,
       "standard": 23.117485762497374
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -62.06896551724138,
       "hard": -92.72727272727272,
       "judge": -87.6923076923077,
       "standard": -85.07462686567165
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 9.068912569956279,
       "hard": -10.76273169840718,
       "judge": 6.830025089827052,
       "standard": 0.43969493062997467
      }
     }
    }
   },
   "jevbench_score": 0.7122309121851964,
   "key": "purplemist-mini-1.0",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "olaverse/PurpleMIST-Mini-1.0@8072361ec4b632b3b8a73899022281d5de3c639b",
   "noul_decisive": {
    "acc_among_decisive": 0.875,
    "decisive": 32,
    "decisive_rate": 0.08579088471849866,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.68645539289165,
     "cost": 70.75879834655666,
     "intelligence": 8.37974071175967,
     "speed": 58.74735259439709
    },
    "board_release": "v1.6.43",
    "capability": 43.033098052325656,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.22686402261042818,
      1.5438120583885713
     ],
     "B": [
      0.1574817264398275,
      1.158827406474195
     ],
     "C": [
      0.15754446014613066,
      1.0720917072142853
     ]
    },
    "jevbench_score": 0.6877481478120564,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": 159,
    "rank": 202,
    "ranks": {
     "A": 202,
     "B": 202,
     "C": 202,
     "capability": 197
    },
    "scores": {
     "A": 0.6877481478120564,
     "B": 0.4967854880080621,
     "C": 0.4776028804250391
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 7.354124985990348,
     "p50_s_raw": 3.6020624929951737,
     "p95_s_adjusted": 18.144029109398247,
     "p95_s_raw": 8.997014554699124
    },
    "views": {
     "Balanced 33:33:33": 0.5599313395330286,
     "Emphasis on Accuracy 60:20:20": 0.3608793205471312,
     "Emphasis on Cost 20:20:60": 0.7856568103657684,
     "Emphasis on Speed 20:60:20": 0.7610524081768362,
     "Intelligence only": 0.23537033944273675
    }
   },
   "rank": 225,
   "ranked": true,
   "ranks": {
    "A": 225,
    "B": 225,
    "C": 225,
    "capability": 218
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/olaverse/PurpleMIST-Mini-1.0",
   "scores": {
    "A": 0.7122309121851964,
    "B": 0.5066847638344534,
    "C": 0.4946048001286087
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3026298979999865,
    "p50_s_raw": 0.07631494899999325,
    "p95_s_adjusted": 0.4624600151999403,
    "p95_s_raw": 0.15623000759997013
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.22727272727272727,
       "cc": -46.57098659210735,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": -100.0,
        "score": -48.04629310965538
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.2871287128712871,
       "cc": -28.74423110677036,
       "cc_by_type": {
        "choice": 10.92258748674444,
        "noul": -100.0,
        "score": 2.844719192944467
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.31683168316831684,
       "cc": -20.82159254385255,
       "cc_by_type": {
        "choice": 15.447154471544714,
        "noul": -85.18518518518519,
        "score": 7.27325308208282
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.375,
       "cc": 0.7629902464480551,
       "cc_by_type": {
        "choice": 42.85714285714286,
        "noul": -60.0,
        "score": 19.431827882201304
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0.7629902464480551
      },
      "jaggedness": {
       "accuracy": 0.26666666666666666,
       "cc": -29.111630061891763,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -100.0,
        "score": -2.156834362768656
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.30990415335463256,
       "cc": -15.461313819540068,
       "cc_by_type": {
        "choice": 22.338645769129176,
        "noul": -71.1340206185567,
        "score": 2.411433390807327
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0.0
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.18518518518518517,
       "cc": -34.84227281771545,
       "cc_by_type": {
        "choice": -11.340206185567009,
        "noul": -100.0,
        "score": 6.8133877324206615
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.36,
       "cc": -16.747368914339727,
       "cc_by_type": {
        "choice": 31.529181610694497,
        "noul": -83.33333333333334,
        "score": 1.562044979619659
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.23255813953488372,
       "cc": -38.366193779640724,
       "cc_by_type": {
        "choice": 6.666666666666665,
        "noul": -93.33333333333333,
        "score": -28.43191467225552
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.3958333333333333,
       "cc": 22.769640479360852,
       "cc_by_type": {
        "choice": 22.769640479360852
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 22.769640479360852
      },
      "perturb_pair": {
       "accuracy": 0.40625,
       "cc": -15.428764708331562,
       "cc_by_type": {
        "choice": 27.92792792792793,
        "noul": -100.0,
        "score": 25.785777947077392
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0.0
      },
      "usecase": {
       "accuracy": 0.25,
       "cc": -24.64856439604378,
       "cc_by_type": {
        "choice": 17.89215686274513,
        "noul": -91.66666666666666,
        "score": -0.1711833842098054
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 1.6816620252144008,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 0.0,
        "score": 0.2830813137384425
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 1.6816620252144008
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.23076923076923078,
       "cc": -23.875193596572927,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -100.0,
        "score": -9.413599222898505
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.5,
       "cc": -8.777693755084039,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": -100.0,
        "score": 13.489927584305406
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "el": {
       "accuracy": 0.06666666666666667,
       "cc": -16.612178963752985,
       "cc_by_type": {
        "choice": -34.07821229050279,
        "noul": -100.0,
        "score": 84.24167539924383
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.2867072111207645,
       "cc": -23.00560999496884,
       "cc_by_type": {
        "choice": 16.74497504301871,
        "noul": -86.33093525179856,
        "score": 0.5691302238733265
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.3870967741935484,
       "cc": -26.26602201719673,
       "cc_by_type": {
        "choice": 21.696574225122355,
        "noul": -75.0,
        "score": -25.49464027671253
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -27.614920679988757,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": 15.515893697738647
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": -8.815298731082237,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": -100.0,
        "score": 15.907044983223884
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.2857142857142857,
       "cc": -32.79394639709711,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -100.0,
        "score": -31.34273304603994
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.4,
       "cc": -23.36579858731335,
       "cc_by_type": {
        "choice": 43.88152766952456,
        "noul": -100.0,
        "score": -13.978923431464608
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ja": {
       "accuracy": 0.4,
       "cc": 0.5067956355677316,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -40.0,
        "score": -1.2913684546947302
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.5067956355677316
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -26.28899664672279,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": 10.02189894872052
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.42857142857142855,
       "cc": -13.785099569294184,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": -100.0,
        "score": -8.021965374549222
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "nl": {
       "accuracy": 0.1,
       "cc": -45.91148210810377,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -100.0,
        "score": -33.838342428207405
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.41025641025641024,
       "cc": -11.560421806782324,
       "cc_by_type": {
        "choice": 54.29538721963606,
        "noul": -80.0,
        "score": -8.97665263998304
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.32142857142857145,
       "cc": -2.352170896253245,
       "cc_by_type": {
        "choice": 4.537418974661164,
        "noul": -19.999999999999996,
        "score": 8.406068336579098
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 1.4666355934971638,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": -100.0,
        "score": 29.931821674108505
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 1.4666355934971638
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -34.881619041434675,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": -17.68833538517358
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": -5.785134578904798,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 47.16072529554367
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -51.780892923305935,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100.0,
        "score": -79.07149232923985
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.38,
       "cc": 19.918100231539075,
       "cc_by_type": {
        "choice": 19.918100231539075
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 19.918100231539075
      },
      "noul": {
       "accuracy": 0.07466666666666667,
       "cc": -85.06666666666666,
       "cc_by_type": {
        "noul": -85.06666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.37333333333333335,
       "cc": 0.5430206313936092,
       "cc_by_type": {
        "score": 0.5430206313936092
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.5430206313936092
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -24.764293855311823,
       "cc_by_type": {
        "choice": 4.0,
        "noul": -75.0,
        "score": -3.2928815659354793
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -25.361393608578854,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100.0,
        "score": -20.183559707724143
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.07142857142857142,
       "cc": -49.2889388162784,
       "cc_by_type": {
        "choice": -13.821138211382115,
        "noul": -100.0,
        "score": -34.0456782374531
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -33.27747339938294,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": 5.430737696588029
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.29411764705882354,
       "cc": -29.86268784562891,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -100.0,
        "score": -17.12429542094469
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.21052631578947367,
       "cc": -15.19949976003395,
       "cc_by_type": {
        "choice": -13.207547169811319,
        "noul": -100.0,
        "score": 67.60904788970947
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.36666666666666664,
       "cc": -12.91912658755326,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -100.0,
        "score": 10.262228080477476
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -32.5134196549177,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100.0,
        "score": -3.463603912488278
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.0,
       "cc": -34.95653361358024,
       "cc_by_type": {
        "choice": -25.0,
        "noul": -100.0,
        "score": 20.13039915925927
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.09090909090909091,
       "cc": -33.2852225600732,
       "cc_by_type": {
        "choice": -15.631691648822272,
        "noul": -66.66666666666667,
        "score": -17.557309364730656
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.32,
       "cc": -11.739809438644576,
       "cc_by_type": {
        "choice": 53.48837209302325,
        "noul": -77.77777777777779,
        "score": -10.930022631179192
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -38.562807098271385,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100.0,
        "score": -18.91422774642706
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.3333333333333333,
       "cc": -22.545583669993736,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100.0,
        "score": 1.4611376656425978
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.3686635944700461,
       "cc": -8.507105822049263,
       "cc_by_type": {
        "choice": 35.71428571428574,
        "noul": -63.63636363636363,
        "score": 2.4007604559301066
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0.0
      },
      "moderation_trust_safety": {
       "accuracy": 0.3888888888888889,
       "cc": -24.616273520558888,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": -16.267246665323487
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.3888888888888889,
       "cc": -17.059669885447803,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -100.0,
        "score": 32.38644995368445
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -20.161455748988658,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -100.0,
        "score": -1.3710667543551436
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -21.852496679566332,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -100.0,
        "score": 35.89178532361983
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.2916666666666667,
       "cc": -24.550847333370474,
       "cc_by_type": {
        "choice": 14.590747330960852,
        "noul": -100.0,
        "score": 11.756710668927727
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.82617742073542,
      81.04552351199263
     ],
     "I": [
      5.614657721176678,
      11.3087412814786
     ],
     "capability": [
      39.544712798455194,
      45.225828017189656
     ],
     "composite": {
      "A": [
       0.2325661652541505,
       1.61193012940895
      ],
      "B": [
       0.15962297864662708,
       1.1885050789163212
      ],
      "C": [
       0.16150428142649342,
       1.1193959232006596
      ]
     },
     "seed": 62
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.68611335416006,
     "capability": 43.0318464940279,
     "intelligence": 8.377579633895749
    },
    "per_type": {
     "choice": {
      "accuracy": 0.38,
      "calibration": 73.99207874399715,
      "cc": 19.918100231539075,
      "cc_by_type": {
       "choice": 19.918100231539075
      },
      "cc_tiered": 20.851483948725576,
      "cc_tiered_open": 21.510322262497397,
      "cc_tiered_sealed": 20.192645634953756,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.918100231539075
     },
     "noul": {
      "accuracy": 0.07466666666666667,
      "calibration": 74.88868920077587,
      "cc": -85.06666666666666,
      "cc_by_type": {
       "noul": -85.06666666666666
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.37333333333333335,
      "calibration": 84.17757211770714,
      "cc": 0.5430206313936092,
      "cc_by_type": {
       "score": 0.5430206313936092
      },
      "cc_tiered": 4.281254952961673,
      "cc_tiered_open": 8.562509905923346,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.5430206313936092
     }
    },
    "raw_on_A": {
     "calibration": 72.87562800357448,
     "capability": 40.88784751944124,
     "intelligence": 8.900067035307991
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.5816619600315053,
    "Emphasis on Accuracy 60:20:20": 0.36599309901574717,
    "Emphasis on Cost 20:20:60": 0.8111015496303313,
    "Emphasis on Speed 20:60:20": 0.8386084205744643,
    "Intelligence only": 0.23518828519516521
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Atlas AI / @leemadov (dr.gaming127@gmail.com)",
   "axes": {
    "calibration": 73.15265267444973,
    "cost": 79.80335744213926,
    "intelligence": 3.96006223937058,
    "speed": 58.9717141639196
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14053714285714305,
      "ece_score": 71.8925714285714,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 71.8925714285714
     },
     "noul": {
      "abstention_rate": 0.9813333333333333,
      "brier": 0.2615201506933332,
      "ece": 0.1349863999999998,
      "n": 375,
      "score": 73.00272000000004
     },
     "score": {
      "interval90_coverage": 1,
      "interval90_width": 0.9808888888888888,
      "n": 375,
      "nrps": 0.19333013477111102,
      "score": 74.56266659477775,
      "top_ece": 0.15770826666666693
     }
    },
    "score": 73.15265267444973
   },
   "capability": 38.556357456910156,
   "composite_ci95": {
    "A": [
     0.010780726312872546,
     0.26774780433532175
    ],
    "B": [
     0.007010377455719735,
     0.18643904716544285
    ],
    "C": [
     0.00748661549505038,
     0.18593597523286234
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public bookable tariff for these weights. The UNMODIFIED frozen pricing_v15 path is asserted below: the declared base Qwen/Qwen3-0.6B is in UNLISTED_BASE_MODELS, so reference() returns no market reference rather than raising, and Florian's standing 24 Sep rule gives a clearly labelled size-class estimate. The sub-1B line of the PUBLISHED v1.6 field is used unchanged - USD 0.01 per 1M input and 0.05 per 1M output, the line liquid-d1-omni-600m (600M), jiwo-0.8b and vegaml-0.8b (0.8B) carry there. No row of THIS draw carries that line; it comes from the earlier draws of the same benchmark - so no new release-lane pricing line is created. x this row's own measured tokens: the author's serve.py returns usage.input_tokens as the length of the one sequence it encodes and usage.output_tokens as a literal 0 (one forward pass, a pointer head, nothing generated). Estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.004712 per 1,000 decisions (0.073x the official cost cap USD 0.064595), against USD 0.006301 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.006301273333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004712175084175084
   },
   "decision_limit": null,
   "display": "Reflex Instinct 0.6B (Atlas AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s into logs/CONDITIONS-<key>.jsonl. The author's own serve.py, shipped in their weights repo, run with their own documented line `python serve.py --port 11472 --device cpu --repo <the pinned local snapshot directory>` inside a loopback-only network namespace with an outbound probe that failed as required, no credentials in the environment, offline (HF_HUB_OFFLINE=1). float32 on CPU, which is their own code's choice for CPU and the right one on Zen 2 (AVX2 only, where bf16 is about 5x slower). Their serve.py sets no torch thread count, so their own default is every core of the host; this run bounded it to 6 of Sandy's 12 (OMP_NUM_THREADS=6) to stay polite to the other jobs on the box - OUR cap, not theirs, and an unbounded host would use all 12. Peak RSS about 7.6 GB, so the row ran in a 14 GiB systemd scope rather than this job's inherited 4 GiB. Speed is therefore a loaded-shared-CPU figure and a lower bound for this model on a dedicated core or a GPU; the standard self-hosted x2 + 0.15 s adjustment is applied on top and the raw p50/p95 are kept. Re-measured for this release on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b) under CoS deputy decision #18209, at USD 0 on Sandy’s own CPU, 6 of 12 threads, starting 2026-10-13 06:14:53Z; the 1-minute load average is logged for the whole window and reported with the row. DISCLOSED: the author’s own encode() truncates whole state lines above its own max_len=16384 rather than refusing them, so this row answers the draw’s 15 items of 106,804–122,936 characters that four other rows of this draw recorded as context refusals (attest-31b-v1.0, julia-1, standard_one_3b_sh and uniphore-4b-decider; krino-2b, both MeTask rows and torchcast-decision-4b-v1.1 answered them) — the author’s design choice, not ours, and those items are the slowest in the row.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.7839203524836424,
    "I_sealed": 1.136204126257517,
    "base": 3.96006223937058,
    "excess": -4.191810544129586,
    "gap": 5.647716226226125,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 20.35176105745093,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 18.27027027027028,
       "hard": 13.180245360176164,
       "judge": 32.76203032300594,
       "standard": 17.12013394725827
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -100,
       "judge": -100,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 15.23909090909089,
       "hard": -5.91657407407411,
       "judge": -12.511694915254235,
       "standard": -44.39370370370372
      }
     },
     "sealed|choice": {
      "cc": 3.4086123787725513,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 25.66709021601017,
       "hard": 1.6421880420955102,
       "judge": 0.39177277179238174,
       "standard": 0.33748154397807795
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -93.10344827586206,
       "hard": -98.18181818181819,
       "judge": -100,
       "standard": -97.01492537313433
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 0.7850877192982497,
       "hard": -17.508647887323825,
       "judge": -3.979895833333358,
       "standard": -4.642090909091023
      }
     }
    }
   },
   "jevbench_score": 0.08485997925452016,
   "key": "reflex-instinct-0.6b",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 (model card; base Qwen/Qwen3-0.6B Apache-2.0)",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.15",
   "measurement_date_status": "measured",
   "method": "v1.6",
   "model_pin": "Atlas-AI-research/reflex-instinct-0.6b@ae04b843e7840d07776c3dc554b4cd064f66cbe5",
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": 0.8571428571428571,
    "decisive": 7,
    "decisive_rate": 0.018666666666666668,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.15265267444973,
     "cost": 79.80335744213926,
     "intelligence": 3.96006223937058,
     "speed": 58.9717141639196
    },
    "board_release": "v1.6.43",
    "capability": 38.556357456910156,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.010780726312872546,
      0.26774780433532175
     ],
     "B": [
      0.007010377455719735,
      0.18643904716544285
     ],
     "C": [
      0.00748661549505038,
      0.18593597523286234
     ]
    },
    "jevbench_score": 0.08485997925452016,
    "measured_in": "v1.6.15",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 0.08485997925452016,
     "B": 0.05721296264713309,
     "C": 0.05893054114897234
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 3.894755522988271,
     "p50_s_raw": 1.8723777614941355,
     "p95_s_adjusted": 32.53481801018057,
     "p95_s_raw": 16.192409005090287
    },
    "views": {
     "Balanced 33:33:33": 0.06673009386029828,
     "Emphasis on Accuracy 60:20:20": 0.039850211293521905,
     "Emphasis on Cost 20:20:60": 0.10213986024727835,
     "Emphasis on Speed 20:60:20": 0.09927765123489675,
     "Intelligence only": 0.02484082563390453
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/Atlas-AI-research/reflex-instinct-0.6b",
   "scores": {
    "A": 0.08485997925452016,
    "B": 0.05721296264713309,
    "C": 0.05893054114897234
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 3.894755522988271,
    "p50_s_raw": 1.8723777614941355,
    "p95_s_adjusted": 32.53481801018057,
    "p95_s_raw": 16.192409005090287
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.22727272727272727,
       "cc": -49.1,
       "cc_by_type": {
        "choice": 0,
        "noul": -100,
        "score": -47.30000000000001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.27722772277227725,
       "cc": -29.19681711735595,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -100,
        "score": -1.6943749999999813
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.19801980198019803,
       "cc": -43.559006670835934,
       "cc_by_type": {
        "choice": 5.691056910569103,
        "noul": -100,
        "score": -36.368076923076906
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.125,
       "cc": -31.862645502645506,
       "cc_by_type": {
        "choice": -14.285714285714285,
        "noul": -100,
        "score": 18.69777777777777
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.2222222222222222,
       "cc": -30.8278251460995,
       "cc_by_type": {
        "choice": 1.8286814244466008,
        "noul": -100,
        "score": 5.6878431372549
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.2779552715654952,
       "cc": -22.6697187396271,
       "cc_by_type": {
        "choice": 26.300959760500138,
        "noul": -87.62886597938144,
        "score": -6.681249999999994
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -53.91467741935484,
       "cc_by_type": {
        "choice": -16.129032258064516,
        "noul": -100,
        "score": -45.61500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -34.40829978726886,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100,
        "score": -28.998095238095246
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.2,
       "cc": -36.25301846655777,
       "cc_by_type": {
        "choice": 1.2063906097163464,
        "noul": -100,
        "score": -9.965446009389645
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.12790697674418605,
       "cc": -57.09688888888889,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": -100,
        "score": -64.62400000000001
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.1875,
       "cc": -3.861517976031954,
       "cc_by_type": {
        "choice": -3.861517976031954
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 0
      },
      "perturb_pair": {
       "accuracy": 0.28125,
       "cc": -41.36431007931008,
       "cc_by_type": {
        "choice": 15.91591591591592,
        "noul": -100,
        "score": -40.00884615384614
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.16363636363636364,
       "cc": -37.397368888888884,
       "cc_by_type": {
        "choice": -4.166666666666629,
        "noul": -100,
        "score": -8.025440000000028
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -34.650927960927966,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -100,
        "score": -27.762307692307697
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.14285714285714285,
       "cc": -52.631578947368425,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.15384615384615385,
       "cc": -36.06443625192013,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -100,
        "score": -25.24400000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.34375,
       "cc": -14.144529947415672,
       "cc_by_type": {
        "choice": 28.318584070796458,
        "noul": -75,
        "score": 4.247826086956518
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.2,
       "cc": -25.974627560521416,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": 5.874999999999986
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.21025195482189402,
       "cc": -33.525042776558344,
       "cc_by_type": {
        "choice": 5.761462515712473,
        "noul": -97.84172661870504,
        "score": -8.494864226682463
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.3870967741935484,
       "cc": -26.126930556888343,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -100,
        "score": -9.865294117647094
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0,
       "cc": -45.96251366120219,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": -6.73999999999999
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.2857142857142857,
       "cc": -35.1146439628483,
       "cc_by_type": {
        "choice": -5.882352941176468,
        "noul": -100,
        "score": 0.538421052631588
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.23809523809523808,
       "cc": -40.884627560521416,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": -38.85500000000002
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.3333333333333333,
       "cc": -29.927109824444205,
       "cc_by_type": {
        "choice": 43.88152766952456,
        "noul": -100,
        "score": -33.66285714285717
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.24,
       "cc": -30.554920483628976,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": -80,
        "score": -11.585333333333313
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -32.21272486772487,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": -7.749285714285725
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.14285714285714285,
       "cc": -33.672222222222224,
       "cc_by_type": {
        "choice": 0,
        "noul": -100,
        "score": -1.0166666666666657
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0,
       "cc": -47.50337662337663,
       "cc_by_type": {
        "choice": -29.87012987012987,
        "noul": -100,
        "score": -12.640000000000029
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.3076923076923077,
       "cc": -20.02176162216602,
       "cc_by_type": {
        "choice": 23.82564536606009,
        "noul": -80,
        "score": -3.890930232558154
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.17857142857142858,
       "cc": -40.33124267366193,
       "cc_by_type": {
        "choice": -6.069534472598708,
        "noul": -100,
        "score": -14.924193548387098
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -30.057553191489365,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100,
        "score": 11.954999999999982
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.15384615384615385,
       "cc": -55.03855072463768,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100,
        "score": -56.42
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -31.236397849462367,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": 3.0649999999999844
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -43.39039548022598,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100,
        "score": -53.90000000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.2786666666666667,
       "cc": 6.829445645726105,
       "cc_by_type": {
        "choice": 6.829445645726105
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 6.829445645726105
      },
      "noul": {
       "accuracy": 0.016,
       "cc": -96.8,
       "cc_by_type": {
        "noul": -96.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.2906666666666667,
       "cc": -9.011182310469268,
       "cc_by_type": {
        "score": -9.011182310469268
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.13636363636363635,
       "cc": -45.081568627451,
       "cc_by_type": {
        "choice": -12,
        "noul": -100,
        "score": -23.24470588235299
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.08333333333333333,
       "cc": -38.55361920688007,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": -100,
        "score": -3.859615384615367
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.21428571428571427,
       "cc": -13.407046070460707,
       "cc_by_type": {
        "choice": -13.821138211382115,
        "noul": -100,
        "score": 73.6
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -32.693250487329436,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100,
        "score": -4.512500000000008
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.11764705882352941,
       "cc": -37.33192632850242,
       "cc_by_type": {
        "choice": -15.94202898550725,
        "noul": -100,
        "score": 3.9462499999999956
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -40.67872117400419,
       "cc_by_type": {
        "choice": -25.786163522012576,
        "noul": -100,
        "score": 3.75000000000002
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.23333333333333334,
       "cc": -25.75370537958774,
       "cc_by_type": {
        "choice": 21.568627450980394,
        "noul": -100,
        "score": 1.1702564102563873
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -33.214854315002775,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": -16.0208695652174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.06666666666666667,
       "cc": -41.89425925925925,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100,
        "score": -21.51611111111109
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.13636363636363635,
       "cc": -42.833514697294135,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -100,
        "score": -25.71681818181817
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.12,
       "cc": -42.82802325581395,
       "cc_by_type": {
        "choice": -16.279069767441868,
        "noul": -100,
        "score": -12.204999999999998
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -28.936626344086022,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -22.293750000000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.2222222222222222,
       "cc": -36.136191263240214,
       "cc_by_type": {
        "choice": 7.869481765834927,
        "noul": -100,
        "score": -16.27805555555557
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.31797235023041476,
       "cc": -17.510075757575745,
       "cc_by_type": {
        "choice": 34.54545454545457,
        "noul": -81.81818181818181,
        "score": -5.257499999999982
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.2222222222222222,
       "cc": -37.63747958300402,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -100,
        "score": -32.29823529411764
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.05555555555555555,
       "cc": -45.341573459038635,
       "cc_by_type": {
        "choice": -16.991643454038996,
        "noul": -100,
        "score": -19.033076923076898
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.16666666666666666,
       "cc": -37.452592485269975,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100,
        "score": 5.868823529411749
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.17391304347826086,
       "cc": -39.18167632850242,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -100,
        "score": -1.603000000000021
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.08333333333333333,
       "cc": -38.66853905225698,
       "cc_by_type": {
        "choice": -17.43772241992883,
        "noul": -100,
        "score": 1.43210526315789
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.93670989622872,
      76.69302559358876
     ],
     "I": [
      1.940386716570835,
      5.948109684014943
     ],
     "capability": [
      35.650836259370074,
      40.70839403134445
     ],
     "composite": {
      "A": [
       0.010780726312872546,
       0.26774780433532175
      ],
      "B": [
       0.007010377455719735,
       0.18643904716544285
      ],
      "C": [
       0.00748661549505038,
       0.18593597523286234
      ]
     },
     "seed": 63
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.15265267444973,
     "capability": 38.556357456910156,
     "intelligence": 3.96006223937058
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2786666666666667,
      "calibration": 71.8925714285714,
      "cc": 6.829445645726105,
      "cc_by_type": {
       "choice": 6.829445645726105
      },
      "cc_tiered": 11.88018671811174,
      "cc_tiered_open": 20.35176105745093,
      "cc_tiered_sealed": 3.4086123787725513,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 6.829445645726105
     },
     "noul": {
      "accuracy": 0.016,
      "calibration": 73.00272000000004,
      "cc": -96.8,
      "cc_by_type": {
       "noul": -96.8
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2906666666666667,
      "calibration": 74.56266659477775,
      "cc": -9.011182310469268,
      "cc_by_type": {
       "score": -9.011182310469268
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 74.53494666952643,
     "capability": 39.32102819546208,
     "intelligence": 4.107109721397716
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.06673009386029828,
    "Emphasis on Accuracy 60:20:20": 0.039850211293521905,
    "Emphasis on Cost 20:20:60": 0.10213986024727835,
    "Emphasis on Speed 20:60:20": 0.09927765123489675,
    "Intelligence only": 0.02484082563390453
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Respan (lihe07)",
   "axes": {
    "calibration": 77.78014902808363,
    "cost": 45.07530479505206,
    "intelligence": 60.897967232257315,
    "speed": 85.7109495365315
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12956210701639312,
      "ece_score": 74.08757859672137,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 74.08757859672137,
      "score_before_time_limit": 74.08757859672137,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.1045576407506702,
      "brier": 0.12399376965004914,
      "ece": 0.09308935506900895,
      "n": 373,
      "score": 81.38212898619821,
      "score_before_time_limit": 81.38212898619821,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8655913978494624,
      "interval90_width": 0.22782258064516128,
      "n": 372,
      "nrps": 0.09086450416692268,
      "score": 77.87073950133129,
      "score_before_time_limit": 77.87073950133129,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.17586035290322588
     }
    },
    "score": 77.78014902808363
   },
   "capability": 69.33905813017047,
   "composite_ci95": {
    "A": [
     49.76126514643758,
     52.669635729769645
    ],
    "B": [
     48.80287113192213,
     52.700909782114465
    ],
    "C": [
     42.23185254314373,
     52.669635729769645
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff for these weights): the frozen market reference google/gemma-4-26b-a4b-it USD 0.09/M input, USD 0.3/M output for the declared base Gemma 4 26B-A4B (catalog id google/gemma-4-26b-a4b-it, supplied by the evaluator because the frozen path matches base names case-sensitively; the submission's model card spells it google/gemma-4-26B-A4B-it) x this row's own measured tokens on the common 1,485-item basis (776,560 input and 102,328 output tokens as reported by the author's server; the 15 refused items count as zero). Estimated, not charged; not a quote from the author or any host. OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.067737 per 1,000 decisions on the common basis (USD 0.067059 over all 1,500 items) is 1.049x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.067737 per 1,000 decisions (1.049x the official cost cap USD 0.064595), against USD 0.067059 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0670592
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06773656565656565
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Respan Span-02",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned rented Lium GPU pod, 1x NVIDIA RTX 5090 32 GB (Romania), NOT the 1x H100 80 GB on which most of this draw's GPU rows were timed: the author's engine runs only on Blackwell (sm_120a), so this row's Speed axis rests on different hardware. The author's own Rust/CUDA dllm-server, built unchanged from their Dockerfile at the pinned revision (weights model.safetensors sha256 5105d39d..., tokenizer cc8d3a0c..., public MTP draft google/gemma-4-26B-A4B-it-assistant pre-staged), started by their own launch.sh with their documented Decision Index settings (container environment HOST=127.0.0.1, PORT=8000; weights mounted read-only at /model, the pre-staged draft at /data) in a container with --network none. Isolation timeline: the container's cut was proven by docker inspect and a failing outbound probe at 05:45:26Z (pod clock) before the sealed input file was copied to the pod host at 05:45:48Z (evaluator clock); the measuring container was then re-created with the same --network none setting and re-proven at 05:45:50Z before the first sealed item was sent at 05:45:55Z. The pod host itself kept its network for the evaluator's ssh session, as for every self-hosted row; no request left the container; one disclosed invocation change: launch.sh is stored without the executable bit in the repository (mode 100644), so the unchanged file was run through the interpreter its own shebang names (/bin/bash) instead of being exec'd directly. Driven by the unchanged run_v15 runner and TypeSafe adapter against /v1/systemone, sequentially, after a 4-item synthetic smoke gate. The standard self-hosted x2 + 0.15 s own-server adjustment (an assumption, not measured) applies on top of the raw p50/p95. 15 of 1,500 requests returned HTTP 400 because prompt + 256-token cap exceeds the server's 16,384-token context: exactly the 15 task ids pre-registered from an offline count with the author's own tokenizer (the same 15 that Attest-31B, Standard One 3B SH, Uniphore-4B-Decider and PurpleMIST-Mini failed on); they are scored as wrong in the full denominator. Measured 14 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b); pod removed afterwards. Regular-queue request GitHub issue #215; CoS deputy #19087/#19088 (spend), custodian #19055 (draw use).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.11779627322474,
    "I_sealed": 55.67813819128989,
    "base": 60.897967232257315,
    "excess": 0.6001313115791405,
    "gap": 10.439658081934851,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 76.60405175662127,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 66.76949620427882,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 70.0,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 54.979840858774125,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.446351153909085,
       "hard": 58.11316922436536,
       "judge": 51.62244140423914,
       "standard": 50.01602816182664
      }
     },
     "sealed|choice": {
      "cc": 74.26722280853451,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 68.77529779114143,
       "judge": 69.14789422135163,
       "standard": 81.01666315123391
      }
     },
     "sealed|noul": {
      "cc": 47.224888338629974,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 34.54545454545455,
       "judge": 38.46153846153846,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 45.542303426705175,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 72.21811261531369,
       "hard": 38.81326566911587,
       "judge": 39.9318705079552,
       "standard": 54.078123725704486
      }
     }
    }
   },
   "jevbench_score": 51.49468169807916,
   "key": "respan-span-02",
   "last_measured_on": "2026-10-14",
   "licence": "Apache-2.0 per model card; weights are a Gemma 4 derivative under the Gemma 4 terms; NOTICE names its vLLM / SGLang / FlashInfer ports (Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "lihe07/span-02@3f0fbdb9c03950f335e80845f51be39985d63fc0",
   "noul_decisive": {
    "acc_among_decisive": 0.8682634730538922,
    "decisive": 334,
    "decisive_rate": 0.8954423592493298,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.78014902808363,
     "cost": 45.07530479505206,
     "intelligence": 60.897967232257315,
     "speed": 85.32322293910049
    },
    "board_release": "v1.6.43",
    "capability": 69.33905813017047,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      49.72091406052094,
      52.62443218967691
     ],
     "B": [
      48.771816312839555,
      52.66469779100122
     ],
     "C": [
      42.19743333671206,
      52.62443218967691
     ]
    },
    "jevbench_score": 51.45147163621532,
    "measured_in": "v1.6.18",
    "method": "v1.6",
    "open_board_rank": 40,
    "rank": 64,
    "ranks": {
     "A": 64,
     "B": 63,
     "C": 50,
     "capability": 83
    },
    "scores": {
     "A": 51.45147163621532,
     "B": 51.04738654109878,
     "C": 51.45147163621532
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20396152500179596,
     "p50_s_raw": 0.02698076250089798,
     "p95_s_adjusted": 1.4392275229169167,
     "p95_s_raw": 0.6446137614584584
    },
    "views": {
     "Balanced 33:33:33": 48.44678117072277,
     "Emphasis on Accuracy 60:20:20": 48.85975361687912,
     "Emphasis on Cost 20:20:60": 42.911545208581245,
     "Emphasis on Speed 20:60:20": 55.08693650119592,
     "Intelligence only": 49.49258431639605
    }
   },
   "rank": 71,
   "ranked": true,
   "ranks": {
    "A": 71,
    "B": 69,
    "C": 55,
    "capability": 97
   },
   "repo": "https://huggingface.co/lihe07/span-02",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 51.49468169807916,
    "B": 51.08140785380785,
    "C": 51.49468169807916
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21189434898260515,
    "p50_s_raw": 0.030947174491302576,
    "p95_s_adjusted": 1.2670264223954288,
    "p95_s_raw": 0.5585132111977145
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 63.655277626666674,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -9.034167119999982
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.655277626666674
      },
      "calibration_gen": {
       "accuracy": 0.7128712871287128,
       "cc": 52.18918897588468,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 28.000000000000004,
        "score": 63.56226470071873
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 52.18918897588468
      },
      "composite_gen": {
       "accuracy": 0.801980198019802,
       "cc": 62.123856121247435,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 33.33333333333333,
        "score": 62.79433259138462
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 62.123856121247435
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.42221393466112,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 99.26664180398333
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.42221393466112
      },
      "jaggedness": {
       "accuracy": 0.7388888888888889,
       "cc": 56.09877202708895,
       "cc_by_type": {
        "choice": 72.56977863330125,
        "noul": 45.945945945945944,
        "score": 49.78059150201963
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 56.09877202708895
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 69.63450226201023,
       "cc_by_type": {
        "choice": 74.64119045522585,
        "noul": 79.38144329896907,
        "score": 54.88087303183574
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.63450226201023
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 36.21711239230192,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 50.0,
        "score": -16.606394781857148
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 36.21711239230192
      },
      "multilingual": {
       "accuracy": 0.7490909090909091,
       "cc": 57.90137613545051,
       "cc_by_type": {
        "choice": 72.61167264427779,
        "noul": 52.77777777777777,
        "score": 48.314677984295976
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.90137613545051
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 49.005011809555555,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 26.66666666666666,
        "score": 20.348368762000003
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 49.005011809555555
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 83.85301706386177,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100.0,
        "score": 78.58607821861231
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.85301706386177
      },
      "usecase": {
       "accuracy": 0.6909090909090909,
       "cc": 53.00694751114175,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 50.0,
        "score": 35.981626847150736
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 53.00694751114175
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 29.139895762588512,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": -93.53269366461541
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 29.139895762588512
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 62.66211428368663,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 8.723670039999964
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 62.66211428368663
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 79.89665532626661,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100.0,
        "score": 79.51297482835739
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 79.89665532626661
      },
      "el": {
       "accuracy": 0.8,
       "cc": 72.17153475017349,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 33.33333333333333,
        "score": 99.94104745349999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 72.17153475017349
      },
      "en": {
       "accuracy": 0.7636837532580365,
       "cc": 60.08592135150014,
       "cc_by_type": {
        "choice": 76.05594269047239,
        "noul": 52.517985611510795,
        "score": 51.68383575251725
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 60.08592135150014
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 50.27799962554451,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": -29.590144679647068
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 50.27799962554451
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 60.1156680291439,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 79.800556
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 60.1156680291439
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 84.44080642965491,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 74.49888987720001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.44080642965491
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 70.13113506375,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75.0,
        "score": 35.39340519125
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.13113506375
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 52.55651539253981,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -14.271217657142854
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 52.55651539253981
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 36.79395242405521,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 19.999999999999996,
        "score": 18.975979591466675
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.79395242405521
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 41.17323553835661,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 67.96415105951428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 41.17323553835661
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.97295969931889,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.91887909795668
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.97295969931889
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -6.864116600293109,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -61.28499049352003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 61.375750592128206,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 51.74468723977861
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.375750592128206
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 57.20084197883275,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 64.6372931727976
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 57.20084197883275
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 52.38715408758865,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 48.22529204999999
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 52.38715408758865
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 79.83776961083333,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 39.513308832499995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 79.83776961083333
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 49.19090294308064,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 12.08883786149999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 49.19090294308064
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 35.897047916258,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -0.21846077100000638
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 35.897047916258
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8133333333333334,
       "cc": 75.88932049981823,
       "cc_by_type": {
        "choice": 75.88932049981823
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.88932049981823
      },
      "noul": {
       "accuracy": 0.7733333333333333,
       "cc": 54.666666666666664,
       "cc_by_type": {
        "noul": 54.666666666666664
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 54.666666666666664
      },
      "score": {
       "accuracy": 0.656,
       "cc": 49.01515268537382,
       "cc_by_type": {
        "score": 49.01515268537382
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 49.01515268537382
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -14.728727007647068,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -50.0,
        "score": -46.186181022941206
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 66.24337792218667,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 17.3636741392308
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 66.24337792218667
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 10.741343942373973,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": -18.995480368000027
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 10.741343942373973
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 3.962179066447362,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -51.27135753750001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 3.962179066447362
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 22.902611299513293,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 31.026674478250005
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.902611299513293
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -27.572823760356403,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -190.89457191000002
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 21.835206443997695,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 9.203098323589732
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.835206443997695
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 24.485053083191342,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50.0,
        "score": 44.36108259452174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 24.485053083191342
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 30.17254713148148,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 33.01764139444444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 30.17254713148148
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 37.54539829080733,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 10.209356899545451
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 37.54539829080733
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 63.9457139474489,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 100.0,
        "score": 3.46504881909091
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.9457139474489
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 59.95658075553763,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 76.64393581499999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 59.95658075553763
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.43503734801719,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 56.821426823322206
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.43503734801719
      },
      "model_routing": {
       "accuracy": 0.8433179723502304,
       "cc": 73.99832376743008,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 96.96969696969697,
        "score": 41.388910696229665
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 73.99832376743008
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 77.61452788840782,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100.0,
        "score": 55.8762132237647
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 77.61452788840782
      },
      "recruiting": {
       "accuracy": 0.9444444444444444,
       "cc": 89.49197567076924,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 68.4759270123077
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 89.49197567076924
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 78.19399701137255,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 34.58199103411764
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 78.19399701137255
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 55.866288543441065,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 66.66666666666667,
        "score": 29.917706210033312
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 55.866288543441065
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.38080365366807,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.1424109610042
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.38080365366807
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.99882158581765,
      80.8791664084684
     ],
     "I": [
      55.16067369419073,
      65.320371533677
     ],
     "capability": [
      64.92778486069663,
      72.55981455085507
     ],
     "composite": {
      "A": [
       49.76126514643758,
       52.669635729769645
      ],
      "B": [
       48.80287113192213,
       52.700909782114465
      ],
      "C": [
       42.23185254314373,
       52.669635729769645
      ]
     },
     "seed": 64
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.78014902808363,
     "capability": 69.33905813017047,
     "intelligence": 60.897967232257315
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8133333333333334,
      "calibration": 74.08757859672137,
      "cc": 75.88932049981823,
      "cc_by_type": {
       "choice": 75.88932049981823
      },
      "cc_tiered": 75.43563728257789,
      "cc_tiered_open": 76.60405175662127,
      "cc_tiered_sealed": 74.26722280853451,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.88932049981823
     },
     "noul": {
      "accuracy": 0.7733333333333333,
      "calibration": 81.38212898619821,
      "cc": 54.666666666666664,
      "cc_by_type": {
       "noul": 54.666666666666664
      },
      "cc_tiered": 56.997192271454395,
      "cc_tiered_open": 66.76949620427882,
      "cc_tiered_sealed": 47.224888338629974,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 54.666666666666664
     },
     "score": {
      "accuracy": 0.656,
      "calibration": 77.87073950133129,
      "cc": 49.01515268537382,
      "cc_by_type": {
       "score": 49.01515268537382
      },
      "cc_tiered": 50.26107214273965,
      "cc_tiered_open": 54.979840858774125,
      "cc_tiered_sealed": 45.542303426705175,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 49.01515268537382
     }
    },
    "raw_on_A": {
     "calibration": 77.50734571344526,
     "capability": 69.8865787495103,
     "intelligence": 62.265811785575366
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 48.49787295439318,
    "Emphasis on Accuracy 60:20:20": 48.89092055818863,
    "Emphasis on Cost 20:20:60": 42.935583645257,
    "Emphasis on Speed 20:60:20": 55.20597038937409,
    "Intelligence only": 49.49258431639605
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "roudone (Hugging Face)",
   "axes": {
    "calibration": 86.57628518330837,
    "cost": 66.19900073922123,
    "intelligence": 27.068221597395038,
    "speed": 91.8816744041786
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0557088695588659,
      "ece_score": 88.85822608822681,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.85822608822681,
      "score_before_time_limit": 88.85822608822681,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.46380697050938335,
      "brier": 0.1673591152201377,
      "ece": 0.0885748488618805,
      "n": 373,
      "score": 82.2850302276239,
      "score_before_time_limit": 82.2850302276239,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.648969534050179,
      "n": 372,
      "nrps": 0.12178463171151906,
      "score": 88.58559923407441,
      "score_before_time_limit": 88.58559923407441,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05325169180349637
     }
    },
    "score": 86.57628518330837
   },
   "capability": 56.822253390351705,
   "composite_ci95": {
    "A": [
     9.817549090172152,
     22.005824456059404
    ],
    "B": [
     7.977868772854861,
     18.84526411413116
    ],
    "C": [
     6.817742423730661,
     15.281822538930143
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 1f4d1f7b1ce97ab27b938043444b9c2757e4a9dbecc799a803122588401f5bf1, 20 Oct additive re-freeze; this base resolves identically under the earlier 6202ef78...): base string Qwen/Qwen3.5-4B (the authors' stated Qwen3.5-4B / NeoHorse lineage, unverified) priced as Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author server's own usage block: 662,682 input on the 1,485 answered items (equal to our offline count through the author's encode() item by item, pre-registered), 0 output tokens (one forward pass with a pointer readout, no generation); the 15 refused items count 0 tokens. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them in full), which this author's server refuses; their tokens count for no row's cost.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.013253640000000002
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013387515151515154
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decider-4B (roudone, NeoHorse 4B pointer model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4 80 GB (FIN-02, Finland, EU, pay-as-you-go non-spot, USD 2.168/h), shared by the two NeoHorse-family rows as two separate back-to-back uses (sealed input deleted from the node between them). The author's own server.py as shipped on stock PyPI vLLM 0.30.0 / torch 2.13.0 in a pre-built image, container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only bundle, all weight shards sha256-verified; driven by the unchanged pinned evaluator driver, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.80120541792978,
    "I_sealed": 21.335237776860296,
    "base": 27.068221597395038,
    "excess": 1.6264408707137719,
    "gap": 11.465967641069483,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 55.18457786363322,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 45.265806857502355,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 8.88198757763975,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": 0.0,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 34.33705081251637,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.81714109054795,
       "hard": 33.27535182513572,
       "judge": 32.334594052853205,
       "standard": 25.22408878775657
      }
     },
     "sealed|choice": {
      "cc": 43.02178780974991,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 58.06861499364677,
       "hard": 39.11183069272579,
       "judge": 44.46620959843293,
       "standard": 41.15165576882515
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -19.999999999999996,
       "judge": -29.230769230769226,
       "standard": 4.477611940298498
      }
     },
     "sealed|score": {
      "cc": 20.98392552083098,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 32.75559389996422,
       "hard": 14.780780553889915,
       "judge": 16.42954167036802,
       "standard": 34.33595704084092
      }
     }
    }
   },
   "jevbench_score": 15.739032004085207,
   "key": "roudone-decider-4b",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (HF metadata and LICENSE), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "roudone/Decider-4B@a9a6a7924f9a6300c43b873df56414c087666083",
   "noul_decisive": {
    "acc_among_decisive": 0.895,
    "decisive": 200,
    "decisive_rate": 0.5361930294906166,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.57628518330837,
     "cost": 66.19900073922123,
     "intelligence": 27.068221597395038,
     "speed": 92.04108849933448
    },
    "board_release": "v1.6.43",
    "capability": 56.822253390351705,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 75,
    "capability_rank": 101,
    "composite_ci95": {
     "A": [
      9.819801762710863,
      22.011766792700506
     ],
     "B": [
      7.979052529165098,
      18.848733187935295
     ],
     "C": [
      6.8193067796603195,
      15.285949161597577
     ]
    },
    "jevbench_score": 15.743016217593315,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": 119,
    "rank": 157,
    "ranks": {
     "A": 157,
     "B": 159,
     "C": 157,
     "capability": 155
    },
    "scores": {
     "A": 15.743016217593315,
     "B": 13.153182490278192,
     "C": 10.932650151106468
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2460378610002408,
     "p50_s_raw": 0.048018930500120405,
     "p95_s_adjusted": 0.25403245620000237,
     "p95_s_raw": 0.05201622810000117
    },
    "views": {
     "Balanced 33:33:33": 13.974968246133828,
     "Emphasis on Accuracy 60:20:20": 10.711684715357173,
     "Emphasis on Cost 20:20:60": 15.735373529584638,
     "Emphasis on Speed 20:60:20": 17.31228143677107,
     "Intelligence only": 7.933031176045365
    }
   },
   "rank": 174,
   "ranked": true,
   "ranks": {
    "A": 174,
    "B": 176,
    "C": 175,
    "capability": 172
   },
   "repo": "https://huggingface.co/roudone/Decider-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.739032004085207,
    "B": 13.150957363067667,
    "C": 10.929883336170283
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24683192200009216,
    "p50_s_raw": 0.04841596100004608,
    "p95_s_adjusted": 0.262682556700247,
    "p95_s_raw": 0.0563412783501235
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server vllm-fast-text/server.py at a9a6a7924f9a6300c43b873df56414c087666083 (HEAD later moved to ee44f389 = README.md only; every .py and weight oid identical), started directly as shipped (independent static review PASS WITH FINDINGS, 0 blockers; no subprocess/network in the server import path; run_evaluation.py, evaluate.py and provenance/* never started). Author defaults kept: position pooling OFF, no prefix or head cache. BASE: the repository's backbone/config.json modification notice and its submission text state that the weights were fine-tuned from Qwen/Qwen3.5-4B (tensor values unchanged by repackaging), while the card's base_model field names TokenRhythm/NeoHorse-1-4B and other metadata names a different NeoHorse name or a local path; the base string Qwen/Qwen3.5-4B is used because it is the upstream model the repository itself states and the only one the frozen pricing reference can resolve. The authors' lineage claims are inconsistent and unverified. FIRST MEASUREMENT (free-queue submission E94B190D; not the unrelated Mapika/decider-4b row). CONTEXT CAP: the author server refuses requests whose state exceeds 8,192 tokens or whose state+question row exceeds 12,288 tokens (HTTP 422); on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like d3-turbo, elf-1-12b and seb-9b on this draw. SAME FAMILY: Goddragon/Arbiter-4B and roudone/Decider-4B are two different checkpoints submitted through the free queue that both serve as 'NeoHorse-Jev-4B' (TYPESAFE_MODEL alias set for the frozen adapter). PROVENANCE, author statements (unverified): base lineage TokenRhythm/NeoHorse-1-4B / Qwen3.5-4B; training on the typed-decisions train split plus other collected decision datasets (README). No rule against public items was found in the reviewed code. HARDWARE: A100 80 GB (the RTX PRO 6000 was out of stock); latency reflects that card. Card metadata names the base inconsistently (NeoHorse-1-4B / NeoHorse-Jev-4B / a local path).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 34.012771811750206,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 50.0,
        "score": -6.295017898082733
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 34.012771811750206
      },
      "calibration_gen": {
       "accuracy": 0.37623762376237624,
       "cc": -4.198761387173277,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -28.000000000000004,
        "score": 1.2997921905480325
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 6.945410137580818,
       "cc_by_type": {
        "choice": 54.47154471544716,
        "noul": -48.14814814814815,
        "score": 14.512833845443451
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 6.945410137580818
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 73.90354996650584,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 61.7106498995175
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 73.90354996650584
      },
      "jaggedness": {
       "accuracy": 0.32222222222222224,
       "cc": -10.471287065762853,
       "cc_by_type": {
        "choice": 16.265640038498567,
        "noul": -56.75675675675676,
        "score": 9.07725552096963
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.7380191693290735,
       "cc": 57.31067969597308,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 60.82474226804124,
        "score": 45.975659943942446
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 57.31067969597308
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -8.559150937831872,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50.0,
        "score": 10.920485330834284
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5418181818181819,
       "cc": 21.37159547449154,
       "cc_by_type": {
        "choice": 48.15780893381155,
        "noul": -22.22222222222222,
        "score": 38.17919971188529
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 21.37159547449154
      },
      "negation_pair_gen": {
       "accuracy": 0.313953488372093,
       "cc": -23.76484736800194,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -86.66666666666667,
        "score": -11.294542104005822
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.7916666666666666,
       "cc": 73.36884154460719,
       "cc_by_type": {
        "choice": 73.36884154460719
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 73.36884154460719
      },
      "perturb_pair": {
       "accuracy": 0.6875,
       "cc": 65.4362645610416,
       "cc_by_type": {
        "choice": 48.94894894894895,
        "noul": 100.0,
        "score": 47.35984473417586
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 65.4362645610416
      },
      "usecase": {
       "accuracy": 0.45454545454545453,
       "cc": 16.25684731361378,
       "cc_by_type": {
        "choice": 41.17647058823531,
        "noul": -16.666666666666664,
        "score": 24.260738019272686
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 16.25684731361378
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 4.470290525637845,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0.0,
        "score": -10.398652232610273
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 4.470290525637845
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": -1.9166212453026752,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -33.333333333333336,
        "score": -10.204548835754412
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 54.14494959379758,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 50.0,
        "score": 44.29325586103874
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 54.14494959379758
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 32.05475206640077,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -33.333333333333336,
        "score": 96.53669567778707
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 32.05475206640077
      },
      "en": {
       "accuracy": 0.5204170286707211,
       "cc": 20.86007070055345,
       "cc_by_type": {
        "choice": 44.64309686237655,
        "noul": -4.316546762589923,
        "score": 22.25366200187372
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 20.86007070055345
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 62.411720719403775,
       "cc_by_type": {
        "choice": 60.848287112561174,
        "noul": 100.0,
        "score": 26.38687504565015
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.411720719403775
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -6.872585540695051,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -33.333333333333336,
        "score": 43.863117694854736
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 44.61722545908915,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 55.028146965502714
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 44.61722545908915
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 3.883462078310893,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50.0,
        "score": 11.929715843871225
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 3.883462078310893
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 1.0,
       "cc": 87.33703584543296,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 62.01110753629888
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 87.33703584543296
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 25.251135116934716,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": 19.999999999999996,
        "score": 41.535772308707244
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 25.251135116934716
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -24.507053960213273,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": 15.367727008249066
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.7857142857142857,
       "cc": 60.47832132317126,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 64.7682973028471
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 60.47832132317126
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 39.71882606567587,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 78.4638375043869
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.71882606567587
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.48717948717948717,
       "cc": 13.214770734569521,
       "cc_by_type": {
        "choice": 54.29538721963606,
        "noul": -40.0,
        "score": 25.348924984072507
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 13.214770734569521
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 23.55856931728997,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 43.71047518816926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 23.55856931728997
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 3.5609835000536996,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 47.278695181012154
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 3.5609835000536996
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 17.000425566473734,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": -5.520462431013584
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 17.000425566473734
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 62.605922524967504,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 52.33389660716057
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.605922524967504
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 9.519007465961737,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -12.68591545522213
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.519007465961737
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.58,
       "cc": 45.750971124590976,
       "cc_by_type": {
        "choice": 45.750971124590976
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 45.750971124590976
      },
      "noul": {
       "accuracy": 0.47733333333333333,
       "cc": -4.533333333333333,
       "cc_by_type": {
        "noul": -4.533333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.488,
       "cc": 25.22129282528254,
       "cc_by_type": {
        "score": 25.22129282528254
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 25.22129282528254
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -24.995524754711237,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -50.0,
        "score": -44.98657426413371
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 4.0585434016706685,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": 1.4095846563577652
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.0585434016706685
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 40.22811272441533,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50.0,
        "score": 35.72498857975006
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 40.22811272441533
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -4.523927283057579,
       "cc_by_type": {
        "choice": -16.959064327485383,
        "noul": -33.333333333333336,
        "score": 36.720615811645985
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": -0.9475624314756766,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100.0,
        "score": 55.12832719832659
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -23.57661018081681,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100.0,
        "score": 17.320483922958374
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.43333333333333335,
       "cc": 8.757336831743325,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 27.11234662968376
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 8.757336831743325
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -18.679188344642103,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 17.133166673390765
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 47.77736546402728,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 60.00000000000001,
        "score": 66.66542972541518
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 47.77736546402728
      },
      "knowledge_graphs": {
       "accuracy": 0.18181818181818182,
       "cc": -10.357701631714406,
       "cc_by_type": {
        "choice": -15.631691648822272,
        "noul": -33.333333333333336,
        "score": 17.89192008701239
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 24.545290011881367,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -11.111111111111116,
        "score": 19.63070207698778
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 24.545290011881367
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -15.709171836735104,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 17.388613522052754
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 27.026718143016733,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 19.999999999999996,
        "score": 18.66172832540338
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 27.026718143016733
      },
      "model_routing": {
       "accuracy": 0.7603686635944701,
       "cc": 60.19956873120693,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 72.72727272727273,
        "score": 35.92338151829608
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 60.19956873120693
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -7.464831114845926,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": 35.187080551815384
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 38.52552177678152,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 15.66941584102237
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 38.52552177678152
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 8.725645628783953,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 5.73358713265727
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 8.725645628783953
      },
      "scientific_discovery": {
       "accuracy": 0.30434782608695654,
       "cc": -15.951830414956177,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -66.66666666666667,
        "score": 20.26045078411697
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 85.08908549209596,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100.0,
        "score": 76.61956964354766
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.08908549209596
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.98866101809301,
      87.30669227235894
     ],
     "I": [
      22.51368521464031,
      31.065507713750073
     ],
     "capability": [
      52.217894521611484,
      58.33910547313941
     ],
     "composite": {
      "A": [
       9.817549090172152,
       22.005824456059404
      ],
      "B": [
       7.977868772854861,
       18.84526411413116
      ],
      "C": [
       6.817742423730661,
       15.281822538930143
      ]
     },
     "seed": 65
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.57628518330837,
     "capability": 56.822253390351705,
     "intelligence": 27.068221597395038
    },
    "per_type": {
     "choice": {
      "accuracy": 0.58,
      "calibration": 88.85822608822681,
      "cc": 45.750971124590976,
      "cc_by_type": {
       "choice": 45.750971124590976
      },
      "cc_tiered": 49.103182836691566,
      "cc_tiered_open": 55.18457786363322,
      "cc_tiered_sealed": 43.02178780974991,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 45.750971124590976
     },
     "noul": {
      "accuracy": 0.47733333333333333,
      "calibration": 82.2850302276239,
      "cc": -4.533333333333333,
      "cc_by_type": {
       "noul": -4.533333333333333
      },
      "cc_tiered": 4.440993788819875,
      "cc_tiered_open": 8.88198757763975,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.488,
      "calibration": 88.58559923407441,
      "cc": 25.22129282528254,
      "cc_by_type": {
       "score": 25.22129282528254
      },
      "cc_tiered": 27.660488166673673,
      "cc_tiered_open": 34.33705081251637,
      "cc_tiered_sealed": 20.98392552083098,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 25.22129282528254
     }
    },
    "raw_on_A": {
     "calibration": 83.84888033371581,
     "capability": 56.08107835039779,
     "intelligence": 28.31327636707977
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 13.9707823645722,
    "Emphasis on Accuracy 60:20:20": 10.710208932865761,
    "Emphasis on Cost 20:20:60": 15.732189091247433,
    "Emphasis on Speed 20:60:20": 17.300722804814377,
    "Intelligence only": 7.933031176045365
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "csabag",
   "axes": {
    "calibration": 73.68644793817732,
    "cost": null,
    "intelligence": 35.358086407070836,
    "speed": 89.07386048376334
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12927287978799915,
      "ece_score": 74.14542404240017,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 73.15681838850149,
      "score_before_time_limit": 74.14542404240017,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.2666666666666667,
      "brier": 0.18679415520437917,
      "ece": 0.15039467431582756,
      "n": 375,
      "score": 69.54815278943805,
      "score_before_time_limit": 69.9210651368345,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.8853333333333333,
      "interval90_width": 0.40755555555555545,
      "n": 375,
      "nrps": 0.12029838922886724,
      "score": 78.35437263659244,
      "score_before_time_limit": 78.98626273850044,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.14998817800056186
     }
    },
    "score": 73.68644793817732
   },
   "capability": 54.52226717262408,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base google/gemma-4-E4B-it (the module RAISES 'base model lacks an explicit market-listing decision'); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's server (input 2,095,834, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RYOTIDE-Gemma (csabag, Gemma-4-E4B stock weights)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own server, next-token softmax over the option markers, T 1.924.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.9529909992295,
    "I_sealed": 28.763181814912173,
    "base": 35.358086407070836,
    "excess": 3.350282413961615,
    "gap": 13.189809184317326,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 57.132088899550936,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965713,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 26.634230503795727,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 42.092653594341826,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.14359632238972,
       "hard": 33.421649824569585,
       "judge": 49.75696822001671,
       "standard": 43.912717831350044
      }
     },
     "sealed|choice": {
      "cc": 50.73087956914682,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 69.50444726810673,
       "hard": 43.79553602405458,
       "judge": 55.04407443682666,
       "standard": 48.74499050833158
      }
     },
     "sealed|noul": {
      "cc": 12.45512490597411,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 0.0,
       "judge": 4.615384615384621,
       "standard": 43.28358208955223
      }
     },
     "sealed|score": {
      "cc": 23.103540969615587,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 51.41157137929402,
       "hard": 9.075703914379096,
       "judge": 24.22274243295568,
       "standard": 35.32639768023921
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "ryotide-gemma",
   "last_measured_on": "2026-10-16",
   "licence": "per base model (stock weights; code licence not checked)",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "csabag/ryotide@6bbb615a170dc16b83ca62373aa7ada7bf3ea27e",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base google/gemma-4-E4B-it and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.8109090909090909,
    "decisive": 275,
    "decisive_rate": 0.7372654155495979,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 74.3509173059117,
     "cost": null,
     "intelligence": 36.06622980750019,
     "speed": 89.15460718023942
    },
    "board_release": "v1.6.43",
    "capability": 55.20857355670594,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3395472984761,
     "p50_s_raw": 0.09477364923804998,
     "p95_s_adjusted": 0.35779890120029445,
     "p95_s_raw": 0.10389945060014724
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://github.com/csabag/ryotide",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.34042168045416477,
    "p50_s_raw": 0.09521084022708237,
    "p95_s_adjusted": 0.3635772834531963,
    "p95_s_raw": 0.10678864172659815
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 78.53492076277824,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 68.93809562166804
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 78.53492076277824
      },
      "calibration_gen": {
       "accuracy": 0.37623762376237624,
       "cc": -3.447323116310777,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -36.0,
        "score": -7.533909963990659
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.48514851485148514,
       "cc": 9.36975394625747,
       "cc_by_type": {
        "choice": 44.71544715447154,
        "noul": -25.92592592592593,
        "score": 9.319740610226802
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 9.36975394625747
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 96.9575143880138,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 90.87254316404142
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 96.9575143880138
      },
      "jaggedness": {
       "accuracy": 0.39444444444444443,
       "cc": 0.6570258968792307,
       "cc_by_type": {
        "choice": 32.14629451395574,
        "noul": -40.54054054054054,
        "score": 10.365323717222497
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.6570258968792307
      },
      "llm_router": {
       "accuracy": 0.7476038338658147,
       "cc": 63.635389162713636,
       "cc_by_type": {
        "choice": 55.622083296645236,
        "noul": 75.25773195876289,
        "score": 60.02635223273278
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.635389162713636
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": 1.9570792379214061,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -25.0,
        "score": 17.46917585809411
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 1.9570792379214061
      },
      "multilingual": {
       "accuracy": 0.6654545454545454,
       "cc": 40.158140045525016,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 13.888888888888884,
        "score": 37.886476797082956
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.158140045525016
      },
      "negation_pair_gen": {
       "accuracy": 0.36046511627906974,
       "cc": -13.691569202634321,
       "cc_by_type": {
        "choice": 23.333333333333332,
        "noul": -46.666666666666664,
        "score": -17.741374274569633
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 64.2793105321989,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 80.0,
        "score": 48.873967632632734
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 64.2793105321989
      },
      "usecase": {
       "accuracy": 0.55,
       "cc": 32.18471870349252,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": 20.833333333333325,
        "score": 22.28945022812462
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 32.18471870349252
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 5.694428754991107,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 0.0,
        "score": 12.321381503068562
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 5.694428754991107
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 10.148014095788994,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -36.22262437929968
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 10.148014095788994
      },
      "de": {
       "accuracy": 0.75,
       "cc": 65.40389985072953,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 75.0,
        "score": 61.03470840174611
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 65.40389985072953
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 38.05525802582019,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 97.77843701973244
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 38.05525802582019
      },
      "en": {
       "accuracy": 0.5708079930495221,
       "cc": 30.292533769567115,
       "cc_by_type": {
        "choice": 49.03650187329906,
        "noul": 16.546762589928065,
        "score": 25.29433684547422
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 30.292533769567115
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 43.345191523632444,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75.0,
        "score": -35.17649720724296
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 43.345191523632444
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 19.61940186440383,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 90.55219466424974
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 19.61940186440383
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 47.804264216420215,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 85.76573382573123
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 47.804264216420215
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 7.619019958927157,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25.0,
        "score": -18.623387050592832
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 7.619019958927157
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 53.98762274993106,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -24.007513667587933
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 53.98762274993106
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 30.888991370728693,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 60.00000000000001,
        "score": 4.152279910438661
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 30.888991370728693
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 28.87078109529692,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 31.056787730335213
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 28.87078109529692
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.46731453674737,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.4019436102421
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.46731453674737
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 39.808499547935135,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 38.04021725852403
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.808499547935135
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 47.21002025003124,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 36.86493167688168
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 47.21002025003124
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 60.43669607801153,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 53.130948575814195
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 60.43669607801153
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -0.9493715267830775,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60.0,
        "score": 59.27954499411885
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 38.182429440216794,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": -35.452711679349626
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 38.182429440216794
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.4457869308839,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 84.85348982490973
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.4457869308839
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 11.977084551443282,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -5.3116841987775
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 11.977084551443282
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.632,
       "cc": 52.46751755678448,
       "cc_by_type": {
        "choice": 52.46751755678448
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 52.46751755678448
      },
      "noul": {
       "accuracy": 0.5946666666666667,
       "cc": 18.933333333333337,
       "cc_by_type": {
        "noul": 18.933333333333337
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.933333333333337
      },
      "score": {
       "accuracy": 0.5173333333333333,
       "cc": 27.879918257891767,
       "cc_by_type": {
        "score": 27.879918257891767
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.879918257891767
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -39.93503064967874,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -75.0,
        "score": -64.80509194903621
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 4.709594579392923,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -11.111111111111116,
        "score": -0.22594366002688382
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.709594579392923
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 68.93000085774041,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": 55.57049037809926
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 68.93000085774041
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -32.72560070376818,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": 7.086355783432308
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 9.09103038460552,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 18.577438979903516
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 9.09103038460552
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -18.430913784732287,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -71.64494261205849
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 24.143316347420846,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 34.89493503666029
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 24.143316347420846
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 31.697117512192023,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 0.0,
        "score": 36.90319922647153
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 31.697117512192023
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 32.0762861885028,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 60.00000000000001,
        "score": 40.395525232175046
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 32.0762861885028
      },
      "knowledge_graphs": {
       "accuracy": 0.45454545454545453,
       "cc": 24.991914766475592,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 66.66666666666667,
        "score": -14.603127934691717
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 24.991914766475592
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 45.57097918413742,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": -7.214711026399101
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 45.57097918413742
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -28.76855468696155,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": -88.45620169529325
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 41.46003313829909,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 27.412728973438515
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 41.46003313829909
      },
      "model_routing": {
       "accuracy": 0.7926267281105991,
       "cc": 64.71215206016127,
       "cc_by_type": {
        "choice": 69.61038961038962,
        "noul": 75.75757575757575,
        "score": 48.768490812518436
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 64.71215206016127
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 37.21060611300256,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 0.0,
        "score": 57.69707745609023
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 37.21060611300256
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 45.50693100770957,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 19.900551611800964
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 45.50693100770957
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 14.94837838181516,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -50.0,
        "score": 39.18011051490362
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 14.94837838181516
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 38.068687261152014,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 38.843742942876325
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 38.068687261152014
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 90.14733599801389,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 81.11816457767156
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 90.14733599801389
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.27452416292749,
      76.49100458899656
     ],
     "I": [
      29.102270255987627,
      39.93644473899713
     ],
     "capability": [
      49.63071921143585,
      57.55172382456721
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 66
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.68644793817732,
     "capability": 54.52226717262408,
     "intelligence": 35.358086407070836
    },
    "per_type": {
     "choice": {
      "accuracy": 0.632,
      "calibration": 73.15681838850149,
      "cc": 52.46751755678448,
      "cc_by_type": {
       "choice": 52.46751755678448
      },
      "cc_tiered": 53.93148423434888,
      "cc_tiered_open": 57.132088899550936,
      "cc_tiered_sealed": 50.73087956914682,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 52.46751755678448
     },
     "noul": {
      "accuracy": 0.5946666666666667,
      "calibration": 69.54815278943805,
      "cc": 18.933333333333337,
      "cc_by_type": {
       "noul": 18.933333333333337
      },
      "cc_tiered": 19.54467770488492,
      "cc_tiered_open": 26.634230503795727,
      "cc_tiered_sealed": 12.45512490597411,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.933333333333337
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 78.35437263659244,
      "cc": 27.879918257891767,
      "cc_by_type": {
       "score": 27.879918257891767
      },
      "cc_tiered": 32.59809728197871,
      "cc_tiered_open": 42.092653594341826,
      "cc_tiered_sealed": 23.103540969615587,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.879918257891767
     }
    },
    "raw_on_A": {
     "calibration": 73.46402121876504,
     "capability": 55.14977186856087,
     "intelligence": 36.8355225183567
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "csabag",
   "axes": {
    "calibration": 87.41983564209619,
    "cost": 62.17918681512502,
    "intelligence": 25.185250272226448,
    "speed": 89.04172792696406
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07791307233311474,
      "ece_score": 84.41738553337706,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 84.41738553337706,
      "score_before_time_limit": 84.41738553337706,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.43466666666666665,
      "brier": 0.15904987256946293,
      "ece": 0.04699539774961242,
      "n": 375,
      "score": 90.60092045007751,
      "score_before_time_limit": 90.60092045007751,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.936,
      "interval90_width": 0.6326666666666665,
      "n": 375,
      "nrps": 0.12350841467172226,
      "score": 87.241200942834,
      "score_before_time_limit": 87.70898218113338,
      "time_limit_failed_share": 0.005333333333333333,
      "top_ece": 0.06115597085280505
     }
    },
    "score": 87.41983564209619
   },
   "capability": 56.30254295716132,
   "composite_ci95": {
    "A": [
     8.805821277560296,
     19.25330176061391
    ],
    "B": [
     7.13766661142721,
     16.483805390793755
    ],
    "C": [
     6.115153664972428,
     13.370348444870773
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base Qwen/Qwen3.5-4B -> deepinfra:Qwen/Qwen3.5-4B USD 0.030000000000000002/M input, USD 0.15/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the author's server (output_tokens 0, nothing generated). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.018226 per 1,000 decisions (0.282x the official cost cap USD 0.064595), against USD 0.041664 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04166444
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.018226101010101012
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RYOTIDE-Qwen (csabag, Qwen3.5-4B stock weights)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own server, next-token softmax over the option markers after 'Answer: **', T 1.0, thinking off.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 27.26180765874407,
    "I_sealed": 23.10869288570883,
    "base": 25.185250272226448,
    "excess": -5.686411997320473,
    "gap": 4.153114773035238,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 52.57893858031065,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 45.265806857502355,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -4.347826086956519,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 29.206484395921564,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.69798528911967,
       "hard": 25.394695936235557,
       "judge": 28.92308053498359,
       "standard": 23.50941666010148
      }
     },
     "sealed|choice": {
      "cc": 50.81225273237945,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 40.67306580316872,
       "judge": 55.04407443682666,
       "standard": 53.4908247205231
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 13.793103448275868,
       "hard": -3.6363636363636376,
       "judge": -23.076923076923073,
       "standard": 19.402985074626855
      }
     },
     "sealed|score": {
      "cc": 18.51382592474705,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 28.34102374356975,
       "hard": 15.694296995745871,
       "judge": 14.062136197193343,
       "standard": 25.916819464668606
      }
     }
    }
   },
   "jevbench_score": 12.935251859192192,
   "key": "ryotide-qwen",
   "last_measured_on": "2026-10-16",
   "licence": "per base model (stock weights; code licence not checked)",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "csabag/ryotide@6bbb615a170dc16b83ca62373aa7ada7bf3ea27e",
   "noul_decisive": {
    "acc_among_decisive": 0.8915094339622641,
    "decisive": 212,
    "decisive_rate": 0.5653333333333334,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.57576272152932,
     "cost": 62.17918681512502,
     "intelligence": 25.4219238453348,
     "speed": 89.1284523402127
    },
    "board_release": "v1.6.43",
    "capability": 56.49884328343206,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 77,
    "capability_rank": 103,
    "composite_ci95": {
     "A": [
      9.09465170059667,
      19.858266669161868
     ],
     "B": [
      7.381986198377254,
      16.979308525255792
     ],
     "C": [
      6.315730347636576,
      13.79046296469574
     ]
    },
    "jevbench_score": 13.247205181296051,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 123,
    "rank": 162,
    "ranks": {
     "A": 162,
     "B": 165,
     "C": 163,
     "capability": 158
    },
    "scores": {
     "A": 13.247205181296051,
     "B": 11.010415184681706,
     "C": 9.199448042566702
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.34223204720765354,
     "p50_s_raw": 0.09611602360382676,
     "p95_s_adjusted": 0.35713637880980964,
     "p95_s_raw": 0.10356818940490484
    },
    "views": {
     "Balanced 33:33:33": 11.63786334339517,
     "Emphasis on Accuracy 60:20:20": 8.895067587552155,
     "Emphasis on Cost 20:20:60": 13.082004734747406,
     "Emphasis on Speed 20:60:20": 14.510305108061395,
     "Intelligence only": 6.571813520246787
    }
   },
   "rank": 182,
   "ranked": true,
   "ranks": {
    "A": 182,
    "B": 187,
    "C": 184,
    "capability": 177
   },
   "repo": "https://github.com/csabag/ryotide",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 12.935251859192192,
    "B": 10.735886118769518,
    "C": 8.982813791105691
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3434839221648872,
    "p50_s_raw": 0.0967419610824436,
    "p95_s_adjusted": 0.36301184310577805,
    "p95_s_raw": 0.10650592155288904
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 60.25604896413942,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 100.0,
        "score": 30.768146892418248
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 60.25604896413942
      },
      "calibration_gen": {
       "accuracy": 0.46534653465346537,
       "cc": 10.945977140386304,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": -12.0,
        "score": -4.260689999837908
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 10.945977140386304
      },
      "composite_gen": {
       "accuracy": 0.43564356435643564,
       "cc": 0.8904059620089105,
       "cc_by_type": {
        "choice": 38.211382113821145,
        "noul": -48.14814814814815,
        "score": 12.607983920353739
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.8904059620089105
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 82.45595791115925,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 86.66666666666667,
        "score": 60.70120706681104
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 82.45595791115925
      },
      "jaggedness": {
       "accuracy": 0.3277777777777778,
       "cc": -10.87940523385924,
       "cc_by_type": {
        "choice": 23.484119345524558,
        "noul": -67.56756756756756,
        "score": 11.445232520465288
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.7092651757188498,
       "cc": 54.43283875070488,
       "cc_by_type": {
        "choice": 59.584397288016206,
        "noul": 64.94845360824742,
        "score": 38.76566535585102
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 54.43283875070488
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -102.05059677779968,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100.0,
        "score": -202.92598388178612
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 17.436943706411615,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": -25.0,
        "score": 14.424233181090518
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 17.436943706411615
      },
      "multilingual": {
       "accuracy": 0.5854545454545454,
       "cc": 29.1324741350392,
       "cc_by_type": {
        "choice": 59.895663514835356,
        "noul": -8.333333333333337,
        "score": 35.83509222361558
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 29.1324741350392
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -21.488753564713154,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -86.66666666666667,
        "score": -4.466260694139468
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.71875,
       "cc": 58.98116200821226,
       "cc_by_type": {
        "choice": 60.96096096096097,
        "noul": 60.00000000000001,
        "score": 55.9825250636758
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 58.98116200821226
      },
      "usecase": {
       "accuracy": 0.45,
       "cc": 15.363822315588715,
       "cc_by_type": {
        "choice": 46.07843137254903,
        "noul": -12.5,
        "score": 12.513035574217113
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 15.363822315588715
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 33.21333936912203,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": -31.31236284501484
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 33.21333936912203
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": 2.2519380465291072,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -33.333333333333336,
        "score": 2.3011290397409345
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 2.2519380465291072
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 73.16070917811867,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 75.0,
        "score": 52.44672930426751
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 73.16070917811867
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 26.27536195081001,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 95.77208212803524
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 26.27536195081001
      },
      "en": {
       "accuracy": 0.526498696785404,
       "cc": 21.683070992286087,
       "cc_by_type": {
        "choice": 48.37749112166067,
        "noul": -1.4388489208633115,
        "score": 18.110570776060907
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 21.683070992286087
      },
      "es": {
       "accuracy": 0.9032258064516129,
       "cc": 75.28442317546406,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 35.641197748251905
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 75.28442317546406
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 18.43017965214669,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 54.19764278157668
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.43017965214669
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 35.04241420570067,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0.0,
        "score": 68.65665438180788
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 35.04241420570067
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 33.036921416788594,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25.0,
        "score": 40.87054078667863
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 33.036921416788594
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 77.12684104141941,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 45.41014120687712
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 77.12684104141941
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 24.835101177285324,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 19.999999999999996,
        "score": 25.990609330108562
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 24.835101177285324
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -25.044405505442313,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": 13.755672372561945
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 70.75737855456181,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 95.6054689970188
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 70.75737855456181
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 31.922090519792775,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 14.380990174096942
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.922090519792775
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 32.62285198021406,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 0.0,
        "score": 20.720862330824154
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 32.62285198021406
      },
      "pt": {
       "accuracy": 0.5714285714285714,
       "cc": 16.92560211002836,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 23.811573566384435
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 16.92560211002836
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 20.77732952898762,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -19.999999999999996,
        "score": 33.39581837419692
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 20.77732952898762
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -7.370688274213745,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0.0,
        "score": -13.416412648728194
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.22817544196927,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 72.20065535816586
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.22817544196927
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 2.656257350195432,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -33.27416580252105
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 2.656257350195432
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.624,
       "cc": 51.434202721062405,
       "cc_by_type": {
        "choice": 51.434202721062405
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 51.434202721062405
      },
      "noul": {
       "accuracy": 0.504,
       "cc": 0.8000000000000007,
       "cc_by_type": {
        "noul": 0.8000000000000007
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.8000000000000007
      },
      "score": {
       "accuracy": 0.43733333333333335,
       "cc": 21.804024593308835,
       "cc_by_type": {
        "score": 21.804024593308835
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 21.804024593308835
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -9.12561345211082,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -50.0,
        "score": -29.37684035633246
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": -0.749620207485151,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": -55.55555555555556,
        "score": -28.059764694229084
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.49853707889798,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 49.015936439945975
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.49853707889798
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -39.558251355212086,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -36.803409036396495
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 12.48460978682998,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -33.333333333333336,
        "score": 43.25093080976531
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 12.48460978682998
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -32.7199888811053,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -100.0,
        "score": -47.845501234510856
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 34.99630093396028,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 20.114953222048914
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.99630093396028
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -24.636887531634997,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100.0,
        "score": 20.165992457359817
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 40.74698882295959,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 24.740966468878767
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 40.74698882295959
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": -0.6958056755951295,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 8.333710739096123
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 16.687683112704796,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": -4.9757103518080825
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 16.687683112704796
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -23.05271961313592,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -4.642029807149695
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.4444444444444444,
       "cc": 12.971072584519268,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -19.999999999999996,
        "score": 16.494791649910965
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 12.971072584519268
      },
      "model_routing": {
       "accuracy": 0.7373271889400922,
       "cc": 55.23694097421215,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 72.72727272727273,
        "score": 21.035498247311768
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 55.23694097421215
      },
      "moderation_trust_safety": {
       "accuracy": 0.3888888888888889,
       "cc": -12.70810171895986,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100.0,
        "score": 30.97358351874422
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 42.400041022365116,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 27.29297357777317
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 42.400041022365116
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": 6.474201484263328,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": 28.535904945400816
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 6.474201484263328
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": 16.615962082396752,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 33.33333333333333,
        "score": 3.4710746529873604
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 16.615962082396752
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 84.82622615418335,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 65.15483504617995
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 84.82622615418335
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.88139116603064,
      88.02962716518056
     ],
     "I": [
      21.652739997621005,
      29.685027913069895
     ],
     "capability": [
      52.06767897446016,
      57.896342508836945
     ],
     "composite": {
      "A": [
       8.805821277560296,
       19.25330176061391
      ],
      "B": [
       7.13766661142721,
       16.483805390793755
      ],
      "C": [
       6.115153664972428,
       13.370348444870773
      ]
     },
     "seed": 67
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.41983564209619,
     "capability": 56.30254295716132,
     "intelligence": 25.185250272226448
    },
    "per_type": {
     "choice": {
      "accuracy": 0.624,
      "calibration": 84.41738553337706,
      "cc": 51.434202721062405,
      "cc_by_type": {
       "choice": 51.434202721062405
      },
      "cc_tiered": 51.69559565634505,
      "cc_tiered_open": 52.57893858031065,
      "cc_tiered_sealed": 50.81225273237945,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.434202721062405
     },
     "noul": {
      "accuracy": 0.504,
      "calibration": 90.60092045007751,
      "cc": 0.8000000000000007,
      "cc_by_type": {
       "noul": 0.8000000000000007
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.8000000000000007
     },
     "score": {
      "accuracy": 0.43733333333333335,
      "calibration": 87.241200942834,
      "cc": 21.804024593308835,
      "cc_by_type": {
       "score": 21.804024593308835
      },
      "cc_tiered": 23.86015516033431,
      "cc_tiered_open": 29.206484395921564,
      "cc_tiered_sealed": 18.51382592474705,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 21.804024593308835
     }
    },
    "raw_on_A": {
     "calibration": 84.62485148494483,
     "capability": 55.81766663420327,
     "intelligence": 27.010481783461714
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 11.357315942111413,
    "Emphasis on Accuracy 60:20:20": 8.663447600523941,
    "Emphasis on Cost 20:20:60": 12.790289399404443,
    "Emphasis on Speed 20:60:20": 14.177335593297395,
    "Intelligence only": 6.389969777013225
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Levanto Labs (marco@levanto.ai)",
   "axes": {
    "calibration": 82.91532974968945,
    "cost": 59.47868179714396,
    "intelligence": 63.700734406993774,
    "speed": 68.89876304791818
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06843028052805278,
      "ece_score": 86.31394389438944,
      "mean_tvd": null,
      "n": 738,
      "n_ece_hard": 303,
      "n_tvd": 0,
      "score": 85.16309130913092,
      "score_before_time_limit": 86.31394389438944,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.08847184986595169,
      "brier": 0.10158675861102966,
      "ece": 0.08940390415549597,
      "n": 373,
      "score": 81.4622654155496,
      "score_before_time_limit": 82.1192191689008,
      "time_limit_failed_share": 0.008
     },
     "score": {
      "interval90_coverage": 0.8897849462365591,
      "interval90_width": 0.2163978494623655,
      "n": 372,
      "nrps": 0.07738047774120042,
      "score": 82.12063252438782,
      "score_before_time_limit": 82.56095763175719,
      "time_limit_failed_share": 0.005333333333333333,
      "top_ece": 0.13570018481182797
     }
    },
    "score": 82.91532974968945
   },
   "capability": 73.30803207834161,
   "composite_ci95": {
    "A": [
     65.47342000127809,
     68.94694303660847
    ],
    "B": [
     63.90184880923016,
     68.72718774008524
    ],
    "C": [
     61.52814597282892,
     68.94694303660847
    ]
   },
   "cost": {
    "basis": "Levanto Labs public list price, USD 0.05/M input, USD 10/M output (fetched 16 Oct 2026): Levanto's public pricing page lists Input USD 0.05 per 1M tokens on its Free and Developer plans (USD 0.046, 0.042 and 0.038 only on the paid monthly volume plans) and \"All plans: Output $10 / 1M tokens\", where output tokens are Sage's reasoning and answers are free (saved copy sha256 1e98749c...). The page states one tariff for the Sage API and does not name a model version. It is the same list basis the published sage-1.3.0 row carries. Counted as the manufacturer's standard, non-promotional launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap and d1-ultra), through a scoped re-freeze of the frozen pricing module that adds exactly three keys (sage-1.4.0, nimble-v4, nimble-v4-max) to its launch-list-price set and nothing else, with a zero-difference proof for every other registry entry and the independent release review; it answers the pricing question that board #18983 left open. The Free plan's included usage is a trial credit and is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (665,989 input tokens over the 1,483 answered items; the provider reported 0 output tokens on every answered item, so the output term is zero as measured), divided by the 1,485 items of the common cost basis the scorer uses; the 17 failed requests (15 HTTP 400, 2 HTTP 504) count as zero tokens and as wrong answers. USD 0.022424 per 1,000 decisions on the common basis (USD 0.022200 over all 1,500 items), 0.347x the official cost cap USD 0.064595. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.022424 per 1,000 decisions (0.347x the official cost cap USD 0.064595), against USD 0.022200 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.022199633333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.022423872053872054
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 13,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Sage 1.4 (Levanto Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted sage.levanto.ai /v1/systemone API, model `levanto-sage-v1.4`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `levanto` before the first request (CoS deputy #18975; every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Regular-queue request 5181ADBA; model id read from the vendor's own /v1/models. 15 of 1,500 requests returned HTTP 400 'Extended context is not available on your plan' with the submitter's own key; they are the draw's 15 longest items (the same 15 that Uniphore-4B-Decider, Attest-31B and Standard One 3B SH failed on). 2 more returned HTTP 504 gateway timeouts. The runner never re-sends a failed item, so all 17 are scored as wrong in the full denominator.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.7993471941146,
    "I_sealed": 60.60212161987294,
    "base": 63.700734406993774,
    "excess": -3.6423011961140475,
    "gap": 6.1972255742416635,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 80.09787615590605,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 63.46997929606625,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 56.830186130371516,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.2471590909091,
       "hard": 63.693576388888886,
       "judge": 56.137447033898304,
       "standard": 37.93402777777778
      }
     },
     "sealed|choice": {
      "cc": 74.0921412438383,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 71.37735630854631,
       "judge": 68.26640548481882,
       "standard": 79.11832946635731
      }
     },
     "sealed|noul": {
      "cc": 60.500725214055116,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 58.18181818181818,
       "judge": 60.00000000000001,
       "standard": 58.20895522388059
      }
     },
     "sealed|score": {
      "cc": 47.2134984017254,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.73396381578947,
       "hard": 39.79533450704228,
       "judge": 41.695149739583336,
       "standard": 58.56711647727272
      }
     }
    }
   },
   "jevbench_score": 67.69827866650344,
   "key": "sage-1.4.0",
   "last_measured_on": "2026-10-13",
   "licence": "Proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:sage.levanto.ai/v1/systemone#levanto-sage-v1.4",
   "noul_decisive": {
    "acc_among_decisive": 0.9050445103857567,
    "decisive": 337,
    "decisive_rate": 0.9108108108108108,
    "n": 375,
    "supported": true,
    "valid": 370
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 19,
    "axes": {
     "calibration": 83.66470689834914,
     "cost": 59.47868179714396,
     "intelligence": 65.88206323551285,
     "speed": 68.80003106294234
    },
    "board_release": "v1.6.43",
    "capability": 74.77338506693098,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      66.41270016475069,
      69.58030491259257
     ],
     "B": [
      65.14310488978832,
      69.5878926976111
     ],
     "C": [
      66.25545115064749,
      69.58030491259257
     ]
    },
    "jevbench_score": 68.40093468849139,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 35,
    "ranks": {
     "A": 35,
     "B": 30,
     "C": 29,
     "capability": 46
    },
    "scores": {
     "A": 68.40093468849139,
     "B": 67.881868670303,
     "C": 68.40093468849139
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.7410984199959785,
     "p50_s_raw": 1.7410984199959785,
     "p95_s_adjusted": 7.571354350852194,
     "p95_s_raw": 7.571354350852194
    },
    "views": {
     "Balanced 33:33:33": 64.47971276851106,
     "Emphasis on Accuracy 60:20:20": 65.03342740838724,
     "Emphasis on Cost 20:20:60": 62.38166597812596,
     "Emphasis on Speed 20:60:20": 66.14105152371843,
     "Intelligence only": 65.88206323551285
    }
   },
   "rank": 44,
   "ranked": true,
   "ranks": {
    "A": 44,
    "B": 37,
    "C": 37,
    "capability": 67
   },
   "repo": "https://levanto.ai/",
   "repriced_in": "v1.6.23",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.69827866650344,
    "B": 66.85912876074042,
    "C": 67.69827866650344
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.705115354496229,
    "p50_s_raw": 1.705115354496229,
    "p95_s_adjusted": 7.5573568566484015,
    "p95_s_raw": 7.5573568566484015
   },
   "status": {
    "answered_ok": 1483,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 77.47395833333333,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 57.421875
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 77.47395833333333
      },
      "calibration_gen": {
       "accuracy": 0.594059405940594,
       "cc": 48.13921763156151,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 76.0,
        "score": 16.1376953125
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 48.13921763156151
      },
      "composite_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 63.04812180066244,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 62.962962962962955,
        "score": 35.937499999999986
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 63.04812180066244
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.10966435185185,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 98.32899305555556
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.10966435185185
      },
      "jaggedness": {
       "accuracy": 0.6722222222222223,
       "cc": 44.68650078063118,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 18.918918918918926,
        "score": 48.34558823529412
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 44.68650078063118
      },
      "llm_router": {
       "accuracy": 0.9201277955271565,
       "cc": 84.76639284962694,
       "cc_by_type": {
        "choice": 91.28290921898389,
        "noul": 97.93814432989691,
        "score": 65.07812500000001
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.76639284962694
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": 15.876531561937488,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25.0,
        "score": 46.8563988095238
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 15.876531561937488
      },
      "multilingual": {
       "accuracy": 0.7236363636363636,
       "cc": 57.13059893713923,
       "cc_by_type": {
        "choice": 67.72089990218454,
        "noul": 44.44444444444444,
        "score": 59.22645246478873
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.13059893713923
      },
      "negation_pair_gen": {
       "accuracy": 0.8604651162790697,
       "cc": 63.684895833333336,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 31.0546875
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 63.684895833333336
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 72.47917087760838,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 80.0,
        "score": 58.45853365384615
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 72.47917087760838
      },
      "usecase": {
       "accuracy": 0.7363636363636363,
       "cc": 62.40265522875817,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 62.5,
        "score": 51.668749999999996
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 62.40265522875817
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 52.014175061050054,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": 25.09014423076922
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 52.014175061050054
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 48.91221038146441,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 34.14062499999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 48.91221038146441
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 83.19280452417597,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100.0,
        "score": 81.4368206521739
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.19280452417597
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 61.01800124146493,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 33.33333333333333,
        "score": 100.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 61.01800124146493
      },
      "en": {
       "accuracy": 0.787141615986099,
       "cc": 65.27214325022268,
       "cc_by_type": {
        "choice": 78.47231544647977,
        "noul": 67.62589928057554,
        "score": 49.71821502361274
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 65.27214325022268
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 64.7293101069955,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 13.763786764705888
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 64.7293101069955
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.919854280510016,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.919854280510016
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 97.08059210526316,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 91.24177631578947
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 97.08059210526316
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 34.37563649324954,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": 25.0,
        "score": 45.166015625
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 34.37563649324954
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 67.14265706769847,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 15.457589285714269
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.14265706769847
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 38.88002217368282,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 19.53125000000001
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.88002217368282
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -14.609168320105821,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": 45.06138392857143
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.7560763888889,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.93489583333334
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.7560763888889
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 5.505163239538234,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 1.7968749999999867
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.505163239538234
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 65.27742887769128,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 60.00000000000001,
        "score": 58.684593023255815
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 65.27742887769128
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 72.55934432838468,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 60.10584677419355
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.55934432838468
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 33.99933510638298,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 58.59375
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 33.99933510638298
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 59.23771512681159,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": 21.19140624999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 59.23771512681159
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.91397849462366,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 75.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.91397849462366
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 26.649305555555554,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 13.28125
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 26.649305555555554
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.816,
       "cc": 76.23375877839223,
       "cc_by_type": {
        "choice": 76.23375877839223
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 76.23375877839223
      },
      "noul": {
       "accuracy": 0.8133333333333334,
       "cc": 62.66666666666667,
       "cc_by_type": {
        "noul": 62.66666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.66666666666667
      },
      "score": {
       "accuracy": 0.648,
       "cc": 51.06646265794228,
       "cc_by_type": {
        "score": 51.06646265794228
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 51.06646265794228
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -14.646752450980395,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -75.0,
        "score": -4.940257352941191
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 65.81281603360408,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100.0,
        "score": 34.705528846153854
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 65.81281603360408
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 53.69452913279133,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100.0,
        "score": -22.65625
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 53.69452913279133
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 9.276391934697855,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": -23.6328125
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 9.276391934697855
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 32.395809744867144,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 73.9990234375
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 32.395809744867144
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 2.590654481132058,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 100.0,
        "score": -154.49218750000006
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 2.590654481132058
      },
      "financial_crime": {
       "accuracy": 0.7333333333333333,
       "cc": 59.81640653056095,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 56.19991987179487
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 59.81640653056095
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 47.58298931980003,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 0.0,
        "score": 63.65489130434783
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 47.58298931980003
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 18.998842592592585,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 18.663194444444432
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.998842592592585
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 61.5794969569896,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 43.76775568181819
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 61.5794969569896
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 43.82631425691019,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 31.995738636363637
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.82631425691019
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -20.121212757616487,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": 2.001953125
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 71.39363736937514,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 77.21354166666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 71.39363736937514
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 82.56224264830222,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100.0,
        "score": 53.5308837890625
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.56224264830222
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 50.763020539309785,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 75.32169117647058
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 50.763020539309785
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 57.781549829774065,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.4375
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.781549829774065
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 48.64196217280016,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50.0,
        "score": 25.482536764705888
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 48.64196217280016
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 40.69802989130435,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 46.73177083333333
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 40.69802989130435
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 97.50205592105264,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 92.5061677631579
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.50205592105264
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.53974842994678,
      84.88171158294799
     ],
     "I": [
      58.130438306540384,
      67.88277812548046
     ],
     "capability": [
      69.0012017096524,
      75.85146696574294
     ],
     "composite": {
      "A": [
       65.47342000127809,
       68.94694303660847
      ],
      "B": [
       63.90184880923016,
       68.72718774008524
      ],
      "C": [
       61.52814597282892,
       68.94694303660847
      ]
     },
     "seed": 68
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.91532974968945,
     "capability": 73.30803207834161,
     "intelligence": 63.700734406993774
    },
    "per_type": {
     "choice": {
      "accuracy": 0.816,
      "calibration": 85.16309130913092,
      "cc": 76.23375877839223,
      "cc_by_type": {
       "choice": 76.23375877839223
      },
      "cc_tiered": 77.09500869987218,
      "cc_tiered_open": 80.09787615590605,
      "cc_tiered_sealed": 74.0921412438383,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.23375877839223
     },
     "noul": {
      "accuracy": 0.8133333333333334,
      "calibration": 81.4622654155496,
      "cc": 62.66666666666667,
      "cc_by_type": {
       "noul": 62.66666666666667
      },
      "cc_tiered": 61.98535225506068,
      "cc_tiered_open": 63.46997929606625,
      "cc_tiered_sealed": 60.500725214055116,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.66666666666667
     },
     "score": {
      "accuracy": 0.648,
      "calibration": 82.12063252438782,
      "cc": 51.06646265794228,
      "cc_by_type": {
       "score": 51.06646265794228
      },
      "cc_tiered": 52.02184226604846,
      "cc_tiered_open": 56.830186130371516,
      "cc_tiered_sealed": 47.2134984017254,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.06646265794228
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.026666666666666616,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.79558015456306,
    "Emphasis on Accuracy 60:20:20": 63.757607983292516,
    "Emphasis on Cost 20:20:60": 61.995746479322406,
    "Emphasis on Speed 20:60:20": 65.74336798715605,
    "Intelligence only": 63.70073440699377
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ironbcc (Hugging Face)",
   "axes": {
    "calibration": 87.96306938738428,
    "cost": 48.88742799396291,
    "intelligence": 38.312942577836566,
    "speed": 90.67341977526846
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.058193423604320185,
      "ece_score": 88.36131527913597,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.36131527913597,
      "score_before_time_limit": 88.36131527913597,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.42627345844504017,
      "brier": 0.12368613516681048,
      "ece": 0.06544694571318402,
      "n": 373,
      "score": 86.9106108573632,
      "score_before_time_limit": 86.9106108573632,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9435483870967742,
      "interval90_width": 0.4249551971326164,
      "n": 372,
      "nrps": 0.09008988865783839,
      "score": 88.61728202565371,
      "score_before_time_limit": 88.61728202565371,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06878223541454365
     }
    },
    "score": 87.96306938738428
   },
   "capability": 63.13800598261042,
   "composite_ci95": {
    "A": [
     22.63082989176838,
     41.67088983143715
    ],
    "B": [
     20.01164308853723,
     38.48846142511844
    ],
    "C": [
     15.715854091505818,
     28.93811793849802
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 6202ef78a2c83cbffc9389cf1f37df70fe0c85ed03f4dd5c5f997f3d31aae13a): base Qwen/Qwen3.5-9B, now declared by the author on the card (revision 4ae564d6) -> qwen/qwen3.5-9b USD 0.09999999999999999/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by vLLM's own usage block: 748,494 input on the 1,485 answered items (equal to our offline per-item count item by item, pre-registered), 1 output token per answered row (one forward pass that reads the label logprobs); the 15 refused items count 0 tokens.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0500481
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.050553636363636356
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Seb-9B (ironbcc, Qwen3.5-9B fine-tune, one forward pass, stock vLLM, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-20",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4 80 GB (FIN-02, Finland, EU, pay-as-you-go non-spot, USD 2.148/h). Stock vLLM 0.30.0 (digest-pinned) + our shim in containers with --network none (cut proven from inside the measuring container before the sealed input), Hub offline, credentials unset, all 20 repo files sha256-verified; driven by the unchanged pinned evaluator driver, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 20 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 44.92534796116673,
    "I_sealed": 31.700537194506396,
    "base": 38.312942577836566,
    "excess": 3.385283996304622,
    "gap": 13.224810766660333,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 69.07949695507523,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 62.25228059138094,
       "judge": 76.26895187870798,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 17.61421670117322,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 13.043478260869556,
       "judge": -9.999999999999998,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 48.082330227251745,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.774986302195465,
       "hard": 46.6348940947645,
       "judge": 48.630522457567935,
       "standard": 42.80858610928006
      }
     },
     "sealed|choice": {
      "cc": 57.06524108576487,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 77.12833545108006,
       "hard": 51.081299872788236,
       "judge": 55.04407443682666,
       "standard": 62.033326302467835
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 37.93103448275863,
       "hard": -14.54545454545455,
       "judge": -10.769230769230765,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 38.03637049775433,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.25874927356028,
       "hard": 29.443486400788153,
       "judge": 33.84604716898835,
       "standard": 46.896434296932675
      }
     }
    }
   },
   "jevbench_score": 32.56208735188496,
   "key": "seb-9b",
   "last_measured_on": "2026-10-20",
   "licence": "Apache-2.0 (LICENSE in the repo and HF metadata; base Qwen/Qwen3.5-9B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.35",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ironbcc/seb-9b@4ae564d62fb69a86d5624c9176c6e271c539485f",
   "noul_decisive": {
    "acc_among_decisive": 0.9392523364485982,
    "decisive": 214,
    "decisive_rate": 0.5737265415549598,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.96306938738428,
     "cost": 48.88742799396291,
     "intelligence": 38.312942577836566,
     "speed": 90.8001817381692
    },
    "board_release": "v1.6.43",
    "capability": 63.13800598261042,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 45,
    "capability_rank": 70,
    "composite_ci95": {
     "A": [
      22.635579367850564,
      41.68054108791833
     ],
     "B": [
      20.014613889486654,
      38.49504774745768
     ],
     "C": [
      15.719152338785115,
      28.944820199943283
     ]
    },
    "jevbench_score": 32.56935976147238,
    "measured_in": "v1.6.35",
    "method": "v1.6",
    "open_board_rank": 65,
    "rank": 95,
    "ranks": {
     "A": 95,
     "B": 97,
     "C": 97,
     "capability": 112
    },
    "scores": {
     "A": 32.56935976147238,
     "B": 29.53085969501141,
     "C": 22.617610945466925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24929025000008095,
     "p50_s_raw": 0.04964512500004048,
     "p95_s_adjusted": 0.3336387865998859,
     "p95_s_raw": 0.09181939329994293
    },
    "views": {
     "Balanced 33:33:33": 29.250716419100634,
     "Emphasis on Accuracy 60:20:20": 25.56750030507671,
     "Emphasis on Cost 20:20:60": 28.498993943725626,
     "Emphasis on Speed 20:60:20": 35.26033540949847,
     "Intelligence only": 21.50556650674145
    }
   },
   "rank": 107,
   "ranked": true,
   "ranks": {
    "A": 107,
    "B": 111,
    "C": 110,
    "capability": 129
   },
   "repo": "https://huggingface.co/ironbcc/seb-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 32.56208735188496,
    "B": 29.526076383243048,
    "C": 22.612560661031225
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2500642110000172,
    "p50_s_raw": 0.050032105500008583,
    "p95_s_adjusted": 0.3424573425000972,
    "p95_s_raw": 0.0962286712500486
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "supersedes_base_row": "v1.6.1",
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "No author code exists in the repository (security scan PASS: no .py, no auto_map, no pickle; config, tokenizer, chat template and preprocessor files carry the same git oids as Qwen/Qwen3.5-9B main). Weights at 4ae564d62fb69a86d5624c9176c6e271c539485f are byte-identical to the 7 Oct measured revision ac57b36b (all four shard LFS sha256 equal; only README.md changed and assets/benchmarks-20261019.json added). Served by the stock vLLM 0.30.0 OpenAI server (image pinned by digest vllm/vllm-openai:v0.30.0@sha256:8a69ffad...) with the author's documented command; the only change from the 7 Oct run is --max-model-len 32768 (was 8192), as the author's current card states. The /v1/systemone wire layer is OUR reviewed shim wire adapter (sha256 e617f911...), unchanged since 7 Oct: it builds the author's documented chat request and renormalises exp(top_logprobs) over the option labels; it applies no temperature and no rule of its own. RE-MEASUREMENT at the author's request (GitHub issue #222): the earlier row (v1.6.1 pool, 7 Oct) lost 23 items to the card's then --max-model-len 8192 and was listed unpriced because no base model was declared. AUTHOR CONTEXT CAP: at the author's current 32,768-token limit vLLM refuses the 15 items of this draw that are 76k-81k tokens long (HTTP 400, mapped to a per-question refusal by our shim); they count as wrong, like d3-turbo and d1-ultra on this draw. OWN HTTP SHIM: the wire layer is ours (see the serving description). TRAINING DATA, the author's own statements (unverified): the card says training included the typed-decisions benchmark's own training split (not its test cases) and that about 45 candidates were compared on the author's own sealed suite before release (selection optimism); issue #222 states that JevBench public items were not used in training or development. All card numbers are self-reported. HARDWARE: measured on an A100 80 GB (the RTX PRO 6000 was out of stock); latency reflects that card. One 85-token warm-up request preceded the scored loop and is not part of any scored row. This re-measurement supersedes the 7 Oct v1.6.1 measurement of the same system on live boards; the frozen v1.6.1 archive retains that row unchanged. The author has declared base Qwen/Qwen3.5-9B since card revision 456cc58; the weights are unchanged from the 7 Oct measurement. SUPERSEDED BY: seb-9b-v2 (/jev-models/seb-9b-v2); the predecessor measurement remains on this board.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 45.223746707381544,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": -30.995426544522033
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 45.223746707381544
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 3.6716931899495946,
       "cc_by_type": {
        "choice": 23.647932131495228,
        "noul": -19.999999999999996,
        "score": 7.36714743835355
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 3.6716931899495946
      },
      "composite_gen": {
       "accuracy": 0.6831683168316832,
       "cc": 39.202518638774364,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -11.111111111111116,
        "score": 57.98695971036105
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 39.202518638774364
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 77.84537293234432,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 73.53611879703296
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.84537293234432
      },
      "jaggedness": {
       "accuracy": 0.40555555555555556,
       "cc": 4.866912866461033,
       "cc_by_type": {
        "choice": 26.37151106833495,
        "noul": -45.945945945945944,
        "score": 34.17517347699409
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 4.866912866461033
      },
      "llm_router": {
       "accuracy": 0.792332268370607,
       "cc": 63.347262987355755,
       "cc_by_type": {
        "choice": 77.01857885004843,
        "noul": 52.57731958762886,
        "score": 60.44589052439
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.347262987355755
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -14.960395410016568,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75.0,
        "score": 4.345617893661635
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6327272727272727,
       "cc": 34.93177969967219,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": -8.333333333333337,
        "score": 47.36408162700266
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.93177969967219
      },
      "negation_pair_gen": {
       "accuracy": 0.5930232558139535,
       "cc": 19.717304053044884,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": -33.333333333333336,
        "score": 15.818578825801332
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 19.717304053044884
      },
      "option_order": {
       "accuracy": 0.7291666666666666,
       "cc": 65.37949400798935,
       "cc_by_type": {
        "choice": 65.37949400798935
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 65.37949400798935
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 74.42814201458951,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 80.0,
        "score": 70.31145307079557
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.42814201458951
      },
      "usecase": {
       "accuracy": 0.5772727272727273,
       "cc": 35.62797906666504,
       "cc_by_type": {
        "choice": 58.33333333333336,
        "noul": 8.333333333333325,
        "score": 40.21727053332845
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 35.62797906666504
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -0.9727306051689837,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -45.775334672649805
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 50.649429931030646,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 60.089610837638816
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 50.649429931030646
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 68.05211316394812,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 53.05014480157888
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.05211316394812
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 27.624535047530582,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 99.81960141819697
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 27.624535047530582
      },
      "en": {
       "accuracy": 0.6211989574283232,
       "cc": 35.82151142994442,
       "cc_by_type": {
        "choice": 58.48232264678242,
        "noul": 7.913669064748197,
        "score": 41.06854257830265
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 35.82151142994442
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 67.2468941424042,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 21.316538870932057
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.2468941424042
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 18.50070873628631,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 54.409230033995534
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.50070873628631
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 73.950788792126,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100.0,
        "score": 85.38177814108387
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 73.950788792126
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 30.07009803937503,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25.0,
        "score": 48.72984719075079
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 30.07009803937503
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 79.07764418870396,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 51.26255064873071
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 79.07764418870396
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 28.371926564196414,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 0.0,
        "score": 28.0069631715408
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 28.371926564196414
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 13.914615064090782,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 41.743845192272346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 13.914615064090782
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.78145276512208,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.0110249620329
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.78145276512208
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 9.24592290572977,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 53.71179469121527
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.24592290572977
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 33.74777631633108,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 36.4781998757812
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 33.74777631633108
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 69.11348793153977,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 49.76827758365878
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 69.11348793153977
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 5.5967829438067485,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 53.3860935122713
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 5.5967829438067485
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 5.8269113862106705,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0.0,
        "score": -60.78013540658538
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 5.8269113862106705
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 70.4456198595988,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 75.8529886110545
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 70.4456198595988
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 7.847467533423753,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -43.1242640663954
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.847467533423753
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.692,
       "cc": 60.21737882470005,
       "cc_by_type": {
        "choice": 60.21737882470005
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 60.21737882470005
      },
      "noul": {
       "accuracy": 0.536,
       "cc": 7.200000000000006,
       "cc_by_type": {
        "noul": 7.200000000000006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 7.200000000000006
      },
      "score": {
       "accuracy": 0.5893333333333334,
       "cc": 41.5783099741794,
       "cc_by_type": {
        "score": 41.5783099741794
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 41.5783099741794
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -11.399722591362355,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -75.0,
        "score": 4.800832225912933
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 13.327689404010428,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 3.406118591603402
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 13.327689404010428
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 52.77491020433876,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50.0,
        "score": 73.36538101952034
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 52.77491020433876
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.8006986415953,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -16.53075089554614
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 27.266093112388443,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 44.11711991687546
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 27.266093112388443
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -52.26868973955401,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": 0.0,
        "score": -168.75575475325323
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 21.869899893472432,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 37.878607243442495
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.869899893472432
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -17.58128304147245,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100.0,
        "score": 30.87984425537359
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 18.174596396111358,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 77.02378918833408
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.174596396111358
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 26.355259817736492,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 0.0,
        "score": 30.457642408241593
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 26.355259817736492
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 46.59774716170659,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 11.111111111111116,
        "score": 28.682130374008665
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 46.59774716170659
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -41.51961222180168,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -62.19324526755556
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 31.68684660421144,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -19.999999999999996,
        "score": 38.09316937117556
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 31.68684660421144
      },
      "model_routing": {
       "accuracy": 0.8341013824884793,
       "cc": 68.73172018279341,
       "cc_by_type": {
        "choice": 84.80519480519482,
        "noul": 72.72727272727273,
        "score": 48.66269301591265
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 68.73172018279341
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 3.1159072574523257,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 55.4129808894395
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.1159072574523257
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 58.39257887745197,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 58.557495221028155
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 58.39257887745197
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 12.422368601973668,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 16.823756052226415
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 12.422368601973668
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 10.835094112201524,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": 42.65020987283646
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 10.835094112201524
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.60108803048422,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.80326409145266
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.60108803048422
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.3070763387534,
      89.00056245181912
     ],
     "I": [
      32.95519131864929,
      42.587296967160135
     ],
     "capability": [
      58.747434360337614,
      65.161057795857
     ],
     "composite": {
      "A": [
       22.63082989176838,
       41.67088983143715
      ],
      "B": [
       20.01164308853723,
       38.48846142511844
      ],
      "C": [
       15.715854091505818,
       28.93811793849802
      ]
     },
     "seed": 69
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.96306938738428,
     "capability": 63.13800598261042,
     "intelligence": 38.312942577836566
    },
    "per_type": {
     "choice": {
      "accuracy": 0.692,
      "calibration": 88.36131527913597,
      "cc": 60.21737882470005,
      "cc_by_type": {
       "choice": 60.21737882470005
      },
      "cc_tiered": 63.07236902042005,
      "cc_tiered_open": 69.07949695507523,
      "cc_tiered_sealed": 57.06524108576487,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.21737882470005
     },
     "noul": {
      "accuracy": 0.536,
      "calibration": 86.9106108573632,
      "cc": 7.200000000000006,
      "cc_by_type": {
       "noul": 7.200000000000006
      },
      "cc_tiered": 8.80710835058661,
      "cc_tiered_open": 17.61421670117322,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 7.200000000000006
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 88.61728202565371,
      "cc": 41.5783099741794,
      "cc_by_type": {
       "score": 41.5783099741794
      },
      "cc_tiered": 43.05935036250304,
      "cc_tiered_open": 48.082330227251745,
      "cc_tiered_sealed": 38.03637049775433,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.5783099741794
     }
    },
    "raw_on_A": {
     "calibration": 84.71000182672668,
     "capability": 62.694073356703875,
     "intelligence": 40.67814488668107
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 29.242895595507655,
    "Emphasis on Accuracy 60:20:20": 25.563914699713497,
    "Emphasis on Cost 20:20:60": 28.49453904306634,
    "Emphasis on Speed 20:60:20": 35.23988561052571,
    "Intelligence only": 21.50556650674145
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "shisa-ai",
   "axes": {
    "calibration": 81.16385348785812,
    "cost": 49.76802917325137,
    "intelligence": 49.90370340622748,
    "speed": 92.69330974783162
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09572306078200861,
      "ece_score": 80.85538784359828,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.74758065980681,
      "score_before_time_limit": 80.85538784359828,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.24128686327077753,
      "brier": 0.1296093675410914,
      "ece": 0.09211681556391924,
      "n": 373,
      "score": 81.57663688721615,
      "score_before_time_limit": 81.57663688721615,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9005376344086021,
      "interval90_width": 0.3373655913978495,
      "n": 372,
      "nrps": 0.10181393375418091,
      "score": 81.1673429165514,
      "score_before_time_limit": 81.1673429165514,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1374196039573955
     }
    },
    "score": 81.16385348785812
   },
   "capability": 65.5337784470428,
   "composite_ci95": {
    "A": [
     46.63348103601634,
     64.24968290106133
    ],
    "B": [
     43.4118482885872,
     61.68741196997996
    ],
    "C": [
     32.3843618305669,
     51.76401817518455
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: declared base google/gemma-4-26b-a4b-it -> google/gemma-4-26b-a4b-it USD 0.09/M input, USD 0.3/M output (frozen at the 25 Sep cut-off, I-3). Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row was first priced with 3948e3f1...d0), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused or failed items cost nothing and still count as wrong answers). Tokens as reported by the evaluator's adapter (prompt tokens of the request(s); output_tokens 0 - the single sampled token is discarded, the answer is the logprob readout). Base string is the frozen module's lower-case key (the card spelling raises). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.047250 per 1,000 decisions (0.731x the official cost cap USD 0.064595), against USD 0.046777 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.046777200000000005
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04724969696969697
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "shisa-de-1 (shisa-ai, Gemma-4-26B-A4B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Stock vLLM server read through the evaluator's shisa_vllm.py adapter.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 61.18017895681861,
    "I_sealed": 41.00946236886275,
    "base": 51.094820662840675,
    "excess": 10.331189817600148,
    "gap": 20.17071658795586,
    "penalty": 0.9766881018239986,
    "per_type_split": {
     "open|choice": {
      "cc": 74.65954284154392,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 67.9144385026738,
       "judge": 68.35860250494397,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 59.19806763285025,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 49.68292639606169,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.25970491817685,
       "hard": 48.94091823006358,
       "judge": 53.33457095279339,
       "standard": 36.90108663190276
      }
     },
     "sealed|choice": {
      "cc": 63.542627740178986,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 57.32624031455995,
       "judge": 63.85896180215475,
       "standard": 61.08415946002954
      }
     },
     "sealed|noul": {
      "cc": 25.330413282034485,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 55.172413793103445,
       "hard": 9.090909090909083,
       "judge": 23.076923076923084,
       "standard": 46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 34.155346084374784,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 59.91120064082291,
       "hard": 23.49152484245691,
       "judge": 33.260433360671,
       "standard": 43.94743037554214
      }
     }
    }
   },
   "jevbench_score": 62.42360135641267,
   "key": "shisa-de-1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "shisa-ai/shisa-de-1@9111f4746a96d6cdf9f1a23a88afd5b57244508b",
   "noul_decisive": {
    "acc_among_decisive": 0.901060070671378,
    "decisive": 283,
    "decisive_rate": 0.7587131367292225,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.19978921578861,
     "cost": 49.76802917325137,
     "intelligence": 49.89769839508435,
     "speed": 92.81832911721798
    },
    "board_release": "v1.6.43",
    "capability": 65.54874380543649,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 36,
    "capability_rank": 57,
    "composite_ci95": {
     "A": [
      46.9264437048365,
      64.2980956074595
     ],
     "B": [
      43.73146315925963,
      61.7198788686833
     ],
     "C": [
      32.587808128358674,
      51.81649202784387
     ]
    },
    "jevbench_score": 62.425924800405696,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 32,
    "rank": 54,
    "ranks": {
     "A": 54,
     "B": 53,
     "C": 60,
     "capability": 99
    },
    "scores": {
     "A": 62.425924800405696,
     "B": 59.25066612002561,
     "C": 43.35133666694839
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2114860150963068,
     "p50_s_raw": 0.0307430075481534,
     "p95_s_adjusted": 0.24710721960291265,
     "p95_s_raw": 0.04855360980145633
    },
    "views": {
     "Balanced 33:33:33": 58.14568090369809,
     "Emphasis on Accuracy 60:20:20": 54.21985915007682,
     "Emphasis on Cost 20:20:60": 54.15770034691774,
     "Emphasis on Speed 20:60:20": 68.08963018332004,
     "Intelligence only": 49.23369255351228
    }
   },
   "rank": 59,
   "ranked": true,
   "ranks": {
    "A": 59,
    "B": 59,
    "C": 70,
    "capability": 115
   },
   "repo": "https://huggingface.co/shisa-ai/shisa-de-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 62.42360135641267,
    "B": 59.25413937791849,
    "C": 43.34972316417546
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21332498043775558,
    "p50_s_raw": 0.03166249021887779,
    "p95_s_adjusted": 0.25213161795400085,
    "p95_s_raw": 0.051065808977000415
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 65.92917797912096,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 6.120867270696206
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 65.92917797912096
      },
      "calibration_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 40.3429206482318,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 19.999999999999996,
        "score": 51.930140523698576
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 40.3429206482318
      },
      "composite_gen": {
       "accuracy": 0.5445544554455446,
       "cc": 21.424225638719204,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -18.518518518518523,
        "score": 12.059488117602957
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 21.424225638719204
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.41167071307218,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 90.23501213921652
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.41167071307218
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 20.239942948437324,
       "cc_by_type": {
        "choice": 55.24542829643889,
        "noul": -8.108108108108103,
        "score": 13.582508656981185
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 20.239942948437324
      },
      "llm_router": {
       "accuracy": 0.805111821086262,
       "cc": 70.45274745673682,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 79.38144329896907,
        "score": 62.88284820393485
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 70.45274745673682
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.5925925925925926,
       "cc": 16.255716229910213,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": -25.0,
        "score": 10.880550751586304
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 16.255716229910213
      },
      "multilingual": {
       "accuracy": 0.7090909090909091,
       "cc": 52.883240812377124,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": 38.888888888888886,
        "score": 50.08362454922064
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 52.883240812377124
      },
      "negation_pair_gen": {
       "accuracy": 0.6511627906976745,
       "cc": 18.559880414922972,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 13.33333333333333,
        "score": -40.98702542189774
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 18.559880414922972
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 78.19555571499772,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100.0,
        "score": 70.6227031810292
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.19555571499772
      },
      "usecase": {
       "accuracy": 0.6181818181818182,
       "cc": 39.882819009244145,
       "cc_by_type": {
        "choice": 64.4607843137255,
        "noul": 20.833333333333325,
        "score": 34.35433938067361
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 39.882819009244145
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 24.773578108952535,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": -56.63164662552334
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.773578108952535
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 51.17586100498403,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 100.0,
        "score": -4.997762607167711
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 51.17586100498403
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 88.18127171952362,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 88.43762046830534
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.18127171952362
      },
      "el": {
       "accuracy": 0.6,
       "cc": 49.62590138419713,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 0.0,
        "score": 99.15703376152992
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 49.62590138419713
      },
      "en": {
       "accuracy": 0.6707211120764552,
       "cc": 44.90753439146581,
       "cc_by_type": {
        "choice": 64.85275991262004,
        "noul": 33.09352517985611,
        "score": 36.776318081921254
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 44.90753439146581
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 61.28889199968085,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": -6.345395779097762
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 61.28889199968085
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 22.258623278201583,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 98.46985890564301
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 22.258623278201583
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 75.251652634066,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 68.10789907866861
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.251652634066
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 52.322008035025284,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25.0,
        "score": 48.725800641388695
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 52.322008035025284
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 51.875208114849244,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -16.315139490214527
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 51.875208114849244
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 29.72773865128922,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 0.0,
        "score": 32.07439943281921
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.72773865128922
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 36.782423082121106,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 54.791713690807754
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 36.782423082121106
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.77110855481779,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.31332566445336
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.77110855481779
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 39.018116704884726,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 35.669068729372796
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.018116704884726
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 45.24014195316485,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 0.0,
        "score": 43.33786132288856
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.24014195316485
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 76.2974441721712,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100.0,
        "score": 50.106239411033336
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.2974441721712
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 34.42500577685311,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 34.33884711779338
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 34.42500577685311
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 61.5543127478497,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50.0,
        "score": 56.40206867833171
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 61.5543127478497
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.41645146226901,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 90.76548341906509
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.41645146226901
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 28.838327496661037,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -46.81835084335022
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 28.838327496661037
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.74,
       "cc": 66.41726783903252,
       "cc_by_type": {
        "choice": 66.41726783903252
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 66.41726783903252
      },
      "noul": {
       "accuracy": 0.68,
       "cc": 36.00000000000001,
       "cc_by_type": {
        "noul": 36.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 36.00000000000001
      },
      "score": {
       "accuracy": 0.5733333333333334,
       "cc": 38.69319904008759,
       "cc_by_type": {
        "score": 38.69319904008759
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 38.69319904008759
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 0.8000025898825479,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -25.0,
        "score": -40.599992230352356
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.8000025898825479
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 42.720119762463035,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 33.33333333333333,
        "score": 13.460566326726564
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 42.720119762463035
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 27.17283322265492,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": -35.96117512878321
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 27.17283322265492
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -14.22728275682919,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -4.085357042417392
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 31.17848786632922,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 41.36155055550942
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 31.17848786632922
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -11.245859971103116,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -33.333333333333336,
        "score": -62.668397523372235
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 35.45895960403414,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": 14.28571428571428,
        "score": 41.110772369525414
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 35.45895960403414
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": -2.1173603271195396,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100.0,
        "score": 45.91272738101071
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 24.81760578587952,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 16.95281735763855
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 24.81760578587952
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 31.67389628464957,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 13.080218475647499
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 31.67389628464957
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 58.062709443665405,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 8.038257529962634
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 58.062709443665405
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 59.72549296402523,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 75.9506724404628
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 59.72549296402523
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 36.95747791084126,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 42.38874851179442
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 36.95747791084126
      },
      "model_routing": {
       "accuracy": 0.8709677419354839,
       "cc": 77.7271211623311,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 96.96969696969697,
        "score": 52.575302880932725
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 77.7271211623311
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 38.84787678865832,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0.0,
        "score": 51.09257470378685
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 38.84787678865832
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 52.02607140073403,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 39.45797279087437
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 52.02607140073403
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 20.93395620361999,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 0.0,
        "score": 21.9151691034708
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 20.93395620361999
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 25.670072643901648,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0.0,
        "score": 34.98123242445856
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 25.670072643901648
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 92.24259919119889,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 87.40395415722656
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.24259919119889
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.8995944023424,
      83.8973143336525
     ],
     "I": [
      44.09204975421906,
      53.92796116673747
     ],
     "capability": [
      61.00415898154735,
      68.10241136551379
     ],
     "composite": {
      "A": [
       46.63348103601634,
       64.24968290106133
      ],
      "B": [
       43.4118482885872,
       61.68741196997996
      ],
      "C": [
       32.3843618305669,
       51.76401817518455
      ]
     },
     "seed": 70
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.16385348785812,
     "capability": 65.5337784470428,
     "intelligence": 49.90370340622748
    },
    "per_type": {
     "choice": {
      "accuracy": 0.74,
      "calibration": 80.74758065980681,
      "cc": 66.41726783903252,
      "cc_by_type": {
       "choice": 66.41726783903252
      },
      "cc_tiered": 69.10108529086145,
      "cc_tiered_open": 74.65954284154392,
      "cc_tiered_sealed": 63.542627740178986,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.41726783903252
     },
     "noul": {
      "accuracy": 0.68,
      "calibration": 81.57663688721615,
      "cc": 36.00000000000001,
      "cc_by_type": {
       "noul": 36.00000000000001
      },
      "cc_tiered": 42.26424045744237,
      "cc_tiered_open": 59.19806763285025,
      "cc_tiered_sealed": 25.330413282034485,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 36.00000000000001
     },
     "score": {
      "accuracy": 0.5733333333333334,
      "calibration": 81.1673429165514,
      "cc": 38.69319904008759,
      "cc_by_type": {
       "score": 38.69319904008759
      },
      "cc_tiered": 41.919136240218236,
      "cc_tiered_open": 49.68292639606169,
      "cc_tiered_sealed": 34.155346084374784,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 38.69319904008759
     }
    },
    "raw_on_A": {
     "calibration": 78.9590048315948,
     "capability": 64.27027081646067,
     "intelligence": 49.581536801326536
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 58.145834497484216,
    "Emphasis on Accuracy 60:20:20": 54.22856181246395,
    "Emphasis on Cost 20:20:60": 54.16353046130678,
    "Emphasis on Speed 20:60:20": 68.06733252370573,
    "Intelligence only": 49.251469993973444
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking",
   "axes": {
    "calibration": 78.35523487828182,
    "cost": 48.64218899335634,
    "intelligence": 32.78619016656576,
    "speed": 86.38605449679788
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12573842746652913,
      "ece_score": 74.85231450669417,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 74.85231450669417,
      "score_before_time_limit": 74.85231450669417,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3726541554959786,
      "brier": 0.17156906643303033,
      "ece": 0.09634973535153724,
      "n": 373,
      "score": 80.73005292969255,
      "score_before_time_limit": 80.73005292969255,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9086021505376344,
      "interval90_width": 0.44130824372759864,
      "n": 372,
      "nrps": 0.12094679093502676,
      "score": 79.48333719845877,
      "score_before_time_limit": 79.48333719845877,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.14469323254789898
     }
    },
    "score": 78.35523487828182
   },
   "capability": 55.57071252242379,
   "composite_ci95": {
    "A": [
     12.825119517917724,
     27.43337606642157
    ],
    "B": [
     11.00758497617656,
     24.829735634421777
    ],
    "C": [
     8.906332998553973,
     19.05095560168164
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference (read 2026-10-12T00:14Z, source/public-docs/OPENROUTER-MINISTRAL-3-PRICES.json): the model card names base_model StandardThinking/StandardOne-3B, built on mistralai/Ministral-3-3B-Instruct-2512. Neither Standard One checkpoint is listed on OpenRouter; the exact underlying base is: OpenRouter mistralai/ministral-3b-2512 list price USD 0.10/M input and USD 0.10/M output. Self-hosted weights, no bookable tariff of the measured deployment; the author's stated hosted list price (USD 0.038/M input, standardthinking.ai/pricing) is below this floor, so the M2 §2 floor binds (price floor: base-model reference price applied). The schema head generates no text, so output tokens are 0 and priced at the reference output price. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.051514 per 1,000 decisions (0.797x the official cost cap USD 0.064595), against USD 0.050999 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05099906666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05151420875420875
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 3B SH (Standard Thinking)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's reference server code/serve_head.py, Quick-start argv, unchanged defaults Paid fast-lane order 3a081599 (public). Same customer and reviewed route as standard_one_8b_sh: official pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime plus the documented requirements (transformers 5.12.1, peft 0.21.0) installed offline from SHA-256-pinned PyPI wheels (PR #299, bound to this order by PR #332). Requests over the documented 32,768-token context answer HTTP 422 (refused, not truncated). Customer list price USD 0.038/M input is below the M2 base floor.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 40.348148949812646,
    "I_sealed": 25.224231383318866,
    "base": 32.78619016656576,
    "excess": 5.284390796138069,
    "gap": 15.12391756649378,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 59.623482293580714,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 52.81535073922617,
       "judge": 48.58272907053395,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 26.282953761214635,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 35.138010794642604,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.381015208512025,
       "hard": 33.253892427920064,
       "judge": 39.06078163000721,
       "standard": 26.90058906810605
      }
     },
     "sealed|choice": {
      "cc": 53.79293995999401,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 79.03430749682339,
       "hard": 40.152654099687744,
       "judge": 56.80705190989228,
       "standard": 63.931659987344446
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -7.272727272727275,
       "judge": -23.076923076923073,
       "standard": 13.432835820895516
      }
     },
     "sealed|score": {
      "cc": 21.879754189962604,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 42.21631474162029,
       "hard": 12.573069905255352,
       "judge": 15.902582759068984,
       "standard": 39.2905996298887
      }
     }
    }
   },
   "jevbench_score": 21.588999608530766,
   "key": "standard_one_3b_sh",
   "last_measured_on": "2026-10-12",
   "licence": "Apache-2.0 (merged weights, schema head and reference server; base Ministral 3 3B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "StandardThinking/StandardOne-3B-SH@bead20e94b7c96045a19dc4d43af48a66c8911ea",
   "noul_decisive": {
    "acc_among_decisive": 0.8418803418803419,
    "decisive": 234,
    "decisive_rate": 0.6273458445040214,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.35523487828182,
     "cost": 48.64218899335634,
     "intelligence": 32.78619016656576,
     "speed": 86.56241878672063
    },
    "board_release": "v1.6.43",
    "capability": 55.57071252242379,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 80,
    "capability_rank": 106,
    "composite_ci95": {
     "A": [
      12.828775574871745,
      27.44229986499225
     ],
     "B": [
      11.009739151550862,
      24.835586726192307
     ],
     "C": [
      8.908871926994268,
      19.057152684022387
     ]
    },
    "jevbench_score": 21.595754998730687,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 86,
    "rank": 119,
    "ranks": {
     "A": 119,
     "B": 133,
     "C": 136,
     "capability": 161
    },
    "scores": {
     "A": 21.595754998730687,
     "B": 19.217967771236903,
     "C": 14.997052082451866
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2202127271797508,
     "p50_s_raw": 0.0351063635898754,
     "p95_s_adjusted": 1.0021107152919284,
     "p95_s_raw": 0.42605535764596425
    },
    "views": {
     "Balanced 33:33:33": 19.4982986853078,
     "Emphasis on Accuracy 60:20:20": 16.460212427864356,
     "Emphasis on Cost 20:20:60": 19.615649720852925,
     "Emphasis on Speed 20:60:20": 23.737566720658748,
     "Intelligence only": 13.341942508299013
    }
   },
   "rank": 135,
   "ranked": true,
   "ranks": {
    "A": 135,
    "B": 151,
    "C": 154,
    "capability": 180
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-3B-SH",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.588999608530766,
    "B": 19.213687633916003,
    "C": 14.992360839257477
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22726393709890544,
    "p50_s_raw": 0.03863196854945272,
    "p95_s_adjusted": 1.0112627825932576,
    "p95_s_raw": 0.43063139129662886
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": " SUPERSEDED BY: standard_one_3b_sh_v4 (/jev-models/standard_one_3b_sh_v4); the predecessor measurement remains on this board.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5454545454545454,
       "cc": 32.77382596686048,
       "cc_by_type": {
        "choice": 25.0,
        "noul": 100.0,
        "score": -26.678522099418565
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 32.77382596686048
      },
      "calibration_gen": {
       "accuracy": 0.594059405940594,
       "cc": 38.90519048770406,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 36.00000000000001,
        "score": 28.435613880927658
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 38.90519048770406
      },
      "composite_gen": {
       "accuracy": 0.6039603960396039,
       "cc": 26.41990183209226,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 3.703703703703698,
        "score": 1.5722619551747141
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 26.41990183209226
      },
      "fairness": {
       "accuracy": 0.6875,
       "cc": 54.227749261100236,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 42.68324778330073
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 54.227749261100236
      },
      "jaggedness": {
       "accuracy": 0.4111111111111111,
       "cc": 4.694910856583905,
       "cc_by_type": {
        "choice": 29.258902791145346,
        "noul": -40.54054054054054,
        "score": 25.366370319146913
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 4.694910856583905
      },
      "llm_router": {
       "accuracy": 0.7092651757188498,
       "cc": 56.50627669788855,
       "cc_by_type": {
        "choice": 59.584397288016206,
        "noul": 60.82474226804124,
        "score": 49.109690537608195
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.50627669788855
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -11.968674887015771,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75.0,
        "score": 13.320779462664035
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6,
       "cc": 27.719617249690184,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": -13.888888888888884,
        "score": 35.195768026286785
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 27.719617249690184
      },
      "negation_pair_gen": {
       "accuracy": 0.5697674418604651,
       "cc": 8.779651370862412,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": -46.666666666666664,
        "score": -10.327712554079426
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 8.779651370862412
      },
      "option_order": {
       "accuracy": 0.5833333333333334,
       "cc": 46.737683089214386,
       "cc_by_type": {
        "choice": 46.737683089214386
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 46.737683089214386
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 65.06712108997921,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 80.0,
        "score": 39.22538729396163
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 65.06712108997921
      },
      "usecase": {
       "accuracy": 0.4909090909090909,
       "cc": 11.828512149644082,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": -37.5,
        "score": 19.554163899912623
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 11.828512149644082
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -8.511711589251624,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 0.0,
        "score": -11.249420482040584
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 50.290968909112884,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 17.539573394005338
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 50.290968909112884
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 77.36447447418108,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 55.987228732277764
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.36447447418108
      },
      "el": {
       "accuracy": 0.4,
       "cc": 20.54630558015352,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 95.34468955237863
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 20.54630558015352
      },
      "en": {
       "accuracy": 0.5716768027801912,
       "cc": 28.048815986073738,
       "cc_by_type": {
        "choice": 53.42990688422155,
        "noul": 6.474820143884896,
        "score": 24.241720930114763
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 28.048815986073738
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 71.78253394019436,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 25.135530042442756
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.78253394019436
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -16.069232069256785,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100.0,
        "score": 82.93984477583622
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": 26.677036311333605,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 22.3840501104714
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 26.677036311333605
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 19.981627998541693,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25.0,
        "score": 18.46443706825078
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 19.981627998541693
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 68.19136996703737,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 18.603727983730966
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.19136996703737
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 22.28425616935985,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": -19.999999999999996,
        "score": 29.743951987031092
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 22.28425616935985
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -12.485068762902236,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 6.989238155737743
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 74.38486199169195,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 89.82125264174256
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 74.38486199169195
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 21.807367811391327,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 24.729462741533283
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 21.807367811391327
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 20.034807741551614,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40.0,
        "score": 22.956729614836824
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 20.034807741551614
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.07017257534663,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 42.03137806781948
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.07017257534663
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 10.403452805008072,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 67.80610309587527
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 10.403452805008072
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 26.71730444108229,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 23.630174192812092
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 26.71730444108229
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 54.56527709507105,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 28.21196031747124
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 54.56527709507105
      },
      "zh": {
       "accuracy": 0.6,
       "cc": -10.14800160355746,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -71.68694266377973
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.656,
       "cc": 55.567462063950714,
       "cc_by_type": {
        "choice": 55.567462063950714
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 55.567462063950714
      },
      "noul": {
       "accuracy": 0.5253333333333333,
       "cc": 5.066666666666664,
       "cc_by_type": {
        "noul": 5.066666666666664
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 5.066666666666664
      },
      "score": {
       "accuracy": 0.488,
       "cc": 26.316988637484396,
       "cc_by_type": {
        "score": 26.316988637484396
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 26.316988637484396
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -6.4142401533742515,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -75.0,
        "score": -12.242720460122758
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 8.17797914598323,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": 13.767891889295447
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 8.17797914598323
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.44150097547703,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 48.844828129683094
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.44150097547703
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -32.50493394907758,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -27.33936325074151
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 15.833160915469014,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 9.818323326117174
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 15.833160915469014
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -48.17502336229969,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100.0,
        "score": -56.47475562149027
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 6.404557415703015,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 20.054008381562827
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 6.404557415703015
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": 1.2325164356394527,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50.0,
        "score": 26.86828101423543
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 1.2325164356394527
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -7.320823604982024,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60.0,
        "score": 21.370862518387256
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.2727272727272727,
       "cc": -4.383667035346825,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -33.333333333333336,
        "score": 22.966058137357102
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 7.379115728815964,
       "cc_by_type": {
        "choice": 53.48837209302325,
        "noul": -33.333333333333336,
        "score": 1.9823084267579727
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 7.379115728815964
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -11.02312325778522,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": 31.4467592589024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.4074074074074074,
       "cc": -0.8936158432810407,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60.0,
        "score": 3.384411587239411
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.7327188940092166,
       "cc": 57.278820382386535,
       "cc_by_type": {
        "choice": 68.44155844155844,
        "noul": 63.63636363636365,
        "score": 39.75853906923753
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 57.278820382386535
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 0.3368320903624114,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 47.07575538816977
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.3368320903624114
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 50.34599830613678,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 34.41775350708257
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.34599830613678
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -12.550772402850038,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -100.0,
        "score": 36.23930840721344
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 5.1704027427711745,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 21.308309677588888
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 5.1704027427711745
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 64.67395698692253,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50.0,
        "score": 54.6980275443975
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 64.67395698692253
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.8126350640683,
      81.09078920907885
     ],
     "I": [
      26.503639969140373,
      36.23806609472571
     ],
     "capability": [
      50.92622576783774,
      58.04236984861399
     ],
     "composite": {
      "A": [
       12.825119517917724,
       27.43337606642157
      ],
      "B": [
       11.00758497617656,
       24.829735634421777
      ],
      "C": [
       8.906332998553973,
       19.05095560168164
      ]
     },
     "seed": 71
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 78.35523487828182,
     "capability": 55.57071252242379,
     "intelligence": 32.78619016656576
    },
    "per_type": {
     "choice": {
      "accuracy": 0.656,
      "calibration": 74.85231450669417,
      "cc": 55.567462063950714,
      "cc_by_type": {
       "choice": 55.567462063950714
      },
      "cc_tiered": 56.70821112678736,
      "cc_tiered_open": 59.623482293580714,
      "cc_tiered_sealed": 53.79293995999401,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.567462063950714
     },
     "noul": {
      "accuracy": 0.5253333333333333,
      "calibration": 80.73005292969255,
      "cc": 5.066666666666664,
      "cc_by_type": {
       "noul": 5.066666666666664
      },
      "cc_tiered": 13.141476880607318,
      "cc_tiered_open": 26.282953761214635,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.066666666666664
     },
     "score": {
      "accuracy": 0.488,
      "calibration": 79.48333719845877,
      "cc": 26.316988637484396,
      "cc_by_type": {
       "score": 26.316988637484396
      },
      "cc_tiered": 28.508882492302604,
      "cc_tiered_open": 35.138010794642604,
      "cc_tiered_sealed": 21.879754189962604,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.316988637484396
     }
    },
    "raw_on_A": {
     "calibration": 76.9738349679489,
     "capability": 52.86740104845798,
     "intelligence": 28.760967128967057
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.49095662175169,
    "Emphasis on Accuracy 60:20:20": 16.457072443157895,
    "Emphasis on Cost 20:20:60": 19.611190631498353,
    "Emphasis on Speed 20:60:20": 23.717988427725025,
    "Intelligence only": 13.341942508299013
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "StartLux Labs",
   "axes": {
    "calibration": 90.90819924197592,
    "cost": 23.51863513364647,
    "intelligence": 66.69999992075427,
    "speed": 75.15747300434305
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03848208596918053,
      "ece_score": 92.3035828061639,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.8112970311977,
      "score_before_time_limit": 92.3035828061639,
      "time_limit_failed_share": 0.005333333333333333
     },
     "noul": {
      "abstention_rate": 0.2278820375335121,
      "brier": 0.06028868195208473,
      "ece": 0.05057905949763694,
      "n": 373,
      "score": 89.64449693220469,
      "score_before_time_limit": 89.88418810047261,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.989247311827957,
      "interval90_width": 0.2383512544802867,
      "n": 372,
      "nrps": 0.041938928585832994,
      "score": 91.2688037625254,
      "score_before_time_limit": 93.004895138443,
      "time_limit_failed_share": 0.018666666666666668,
      "top_ece": 0.04898158432265336
     }
    },
    "score": 90.90819924197592
   },
   "capability": 78.8040995813651,
   "composite_ci95": {
    "A": [
     10.629002654265207,
     10.928544162470537
    ],
    "B": [
     11.114542762869075,
     11.645757487636661
    ],
    "C": [
     10.629002654265207,
     10.928544162470537
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: base Qwen/Qwen3.8-27B -> qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output. Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 3948e3f1...ea4d0, the module this row was first priced with (the later re-freeze only adds a hosted-API launch-list key), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). The model cards do not name the Qwen version; Qwen/Qwen3.8-27B is the base the Decision Index metadata gives for the sibling Instant variant (AutoThink itself is not on the Decision Index), shares the same NOTICE, consistent with the NOTICE (modified Alibaba Cloud Apache-2.0 model) and with the config (Qwen3_5ForConditionalGeneration, 64 layers, hidden 5120), the same reference as the live torchcast-decision-27b row (a fine-tune of StartLux-Decision-27B v1). OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.354306 per 1,000 decisions on the common basis (USD 0.350763 over all 1,500 items) is 5.485x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.354306 per 1,000 decisions (5.485x the official cost cap USD 0.064595), against USD 0.350763 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.3507632
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.35430626262626264
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 12,
    "n_over_limit_answered_ok": 12,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "StartLux-Decision v1.1 27B AutoThink (StartLux Labs, Qwen3.8-27B fine-tune, gate 0.6)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H200 141 GB (sm_90, driver 580.173.02), Finland (EU). The author's own server on vLLM 0.19.1, offline, loopback, one request at a time.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.83827136985272,
    "I_sealed": 65.56172847165581,
    "base": 66.69999992075427,
    "excess": -7.562983872158803,
    "gap": 2.276542898196908,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 94.7183214036337,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 90.56307014784524,
       "judge": 100.0,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 47.46790890269151,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 65.21739130434783,
       "judge": 30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 61.328583803232945,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.57481560088962,
       "hard": 84.34403756537992,
       "judge": 31.19020107738477,
       "standard": 57.38213446888292
      }
     },
     "sealed|choice": {
      "cc": 88.15105310694662,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 83.34682548860874,
       "judge": 90.30362389813908,
       "standard": 89.55916473317865
      }
     },
     "sealed|noul": {
      "cc": 46.833366324874305,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 58.62068965517242,
       "hard": 39.99999999999999,
       "judge": 38.46153846153846,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 61.700765983146525,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 76.7463644799391,
       "hard": 54.79544186930036,
       "judge": 65.8141729534546,
       "standard": 61.81850450698043
      }
     }
    }
   },
   "jevbench_score": 10.816782537639455,
   "key": "startlux-decision-v1.1-27b-autothink",
   "last_measured_on": "2026-10-16",
   "licence": "Weights CC BY-NC 4.0 (non-commercial: commercial use needs a separate licence from StartLux Labs; listing a measured score is evaluation use); inference package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "startlux-models/StartLux-Decision-v1.1-27B-AutoThink@dd62d5cd84a483d75bdd548880bbf08bdd3f67a4",
   "noul_decisive": {
    "acc_among_decisive": 0.96875,
    "decisive": 288,
    "decisive_rate": 0.7741935483870968,
    "n": 375,
    "supported": true,
    "valid": 372
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.73088868169317,
     "cost": 23.51863513364647,
     "intelligence": 68.96369650479022,
     "speed": 75.05228443363347
    },
    "board_release": "v1.6.43",
    "capability": 80.34729259324169,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      10.723171777210583,
      10.982898737896896
     ],
     "B": [
      11.285116241749748,
      11.74017776896854
     ],
     "C": [
      10.723171777210583,
      10.982898737896896
     ]
    },
    "jevbench_score": 10.892954017975724,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": 129,
    "rank": 168,
    "ranks": {
     "A": 168,
     "B": 162,
     "C": 158,
     "capability": 23
    },
    "scores": {
     "A": 10.892954017975724,
     "B": 11.554064135977844,
     "C": 10.892954017975724
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26122978397179397,
     "p50_s_raw": 0.055614891985896975,
     "p95_s_adjusted": 11.960487055202247,
     "p95_s_raw": 5.905243527601123
    },
    "views": {
     "Balanced 33:33:33": 9.435806388573084,
     "Emphasis on Accuracy 60:20:20": 11.135498515443802,
     "Emphasis on Cost 20:20:60": 7.119526996383961,
     "Emphasis on Speed 20:60:20": 11.405605564551664,
     "Intelligence only": 15.258250914248908
    }
   },
   "rank": 190,
   "ranked": true,
   "ranks": {
    "A": 190,
    "B": 183,
    "C": 177,
    "capability": 37
   },
   "repo": "https://huggingface.co/startlux-models/StartLux-Decision-v1.1-27B-AutoThink",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 10.816782537639455,
    "B": 11.42704836528611,
    "C": 10.816782537639455
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28974409399088474,
    "p50_s_raw": 0.06987204699544236,
    "p95_s_adjusted": 10.52538783007183,
    "p95_s_raw": 5.187693915035915
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 91.36974107629146,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 82.4425565622077
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 91.36974107629146
      },
      "calibration_gen": {
       "accuracy": 0.7524752475247525,
       "cc": 55.45797264538123,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 36.00000000000001,
        "score": 59.00594338683297
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 55.45797264538123
      },
      "composite_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 53.48759110244231,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 3.703703703703698,
        "score": 63.26313464427364
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 53.48759110244231
      },
      "fairness": {
       "accuracy": 0.9583333333333334,
       "cc": 88.80841269511244,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 73.33333333333334,
        "score": 93.09190475200394
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 88.80841269511244
      },
      "jaggedness": {
       "accuracy": 0.8222222222222222,
       "cc": 58.727647366249755,
       "cc_by_type": {
        "choice": 91.33782483156881,
        "noul": 29.729729729729737,
        "score": 55.11538753745072
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.727647366249755
      },
      "llm_router": {
       "accuracy": 0.9169329073482428,
       "cc": 79.845307649839,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 87.62886597938144,
        "score": 57.45429655805487
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 79.845307649839
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 56.54793730611436,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 25.0,
        "score": 69.3860799595802
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 56.54793730611436
      },
      "multilingual": {
       "accuracy": 0.8581818181818182,
       "cc": 70.68658003787642,
       "cc_by_type": {
        "choice": 91.19660906423215,
        "noul": 44.44444444444444,
        "score": 76.41868660495271
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.68658003787642
      },
      "negation_pair_gen": {
       "accuracy": 0.7209302325581395,
       "cc": 38.055841143078034,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -26.66666666666667,
        "score": 40.834190095900766
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 38.055841143078034
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 70.62365287202225,
       "cc_by_type": {
        "choice": 90.99099099099098,
        "noul": 80.0,
        "score": 40.87996762507578
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.62365287202225
      },
      "usecase": {
       "accuracy": 0.8590909090909091,
       "cc": 71.9252923706649,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 62.5,
        "score": 63.07979868062215
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 71.9252923706649
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 61.094538064721405,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": 52.33123324178327
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 61.094538064721405
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 73.25755504785428,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 86.43933181022953
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 73.25755504785428
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.14337867842146,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 76.32394134499887
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.14337867842146
      },
      "el": {
       "accuracy": 0.8,
       "cc": 72.15987140852917,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 33.33333333333333,
        "score": 99.90605742856702
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 72.15987140852917
      },
      "en": {
       "accuracy": 0.8410078192875761,
       "cc": 65.6279430221757,
       "cc_by_type": {
        "choice": 89.23615772323987,
        "noul": 47.482014388489205,
        "score": 60.16565695479803
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 65.6279430221757
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 71.39927626489069,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 23.98575701653176
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.39927626489069
      },
      "fi": {
       "accuracy": 1.0,
       "cc": 99.45381474863512,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.36144424590536
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 99.45381474863512
      },
      "fr": {
       "accuracy": 1.0,
       "cc": 98.79708726840215,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.39126180520648
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 98.79708726840215
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 68.26841937256562,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 25.0,
        "score": 79.80525811769688
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 68.26841937256562
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 74.74473760523466,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 24.23421281570397
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 74.74473760523466
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 54.038814370488296,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 39.99999999999999,
        "score": 22.11644311146491
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 54.038814370488296
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 14.500731144311763,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 43.502193432935286
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 14.500731144311763
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.20107791961944,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 98.26990042552501
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.20107791961944
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 50.22874987381278,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 84.01958295477169
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 50.22874987381278
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 61.29679898233318,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 19.999999999999996,
        "score": 71.50783241039356
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.29679898233318
      },
      "pt": {
       "accuracy": 0.8928571428571429,
       "cc": 86.54314095373468,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100.0,
        "score": 80.84332975572379
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 86.54314095373468
      },
      "sv": {
       "accuracy": 0.9090909090909091,
       "cc": 87.36909613502253,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 100.0,
        "score": 87.63920329868462
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 87.36909613502253
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 75.97905009330604,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 77.93715027991811
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 75.97905009330604
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.46841338981227,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 94.66330468556585
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.46841338981227
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 48.76376745279072,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 12.957969025038828
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.76376745279072
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.9186666666666666,
       "cc": 89.49463250349223,
       "cc_by_type": {
        "choice": 89.49463250349223
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 89.49463250349223
      },
      "noul": {
       "accuracy": 0.744,
       "cc": 48.8,
       "cc_by_type": {
        "noul": 48.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 48.8
      },
      "score": {
       "accuracy": 0.7973333333333333,
       "cc": 62.42466072114584,
       "cc_by_type": {
        "score": 62.42466072114584
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 62.42466072114584
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 34.51865055496734,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 0.0,
        "score": 19.555951664902004
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 34.51865055496734
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 80.62020123707451,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 64.08282593344576
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.62020123707451
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 67.14209875854465,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50.0,
        "score": 67.68645887725995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 67.14209875854465
      },
      "demand_forecasting": {
       "accuracy": 0.8235294117647058,
       "cc": 62.88499771485198,
       "cc_by_type": {
        "choice": 76.60818713450293,
        "noul": 33.33333333333333,
        "score": 78.71347267671969
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 62.88499771485198
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.9411764705882353,
       "cc": 89.9509753012271,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 69.8529259036813
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 89.9509753012271
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 29.476842130779346,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 66.66666666666667,
        "score": -65.65752392212738
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 29.476842130779346
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 69.04160611650587,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 42.85714285714286,
        "score": 64.26767549237475
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 69.04160611650587
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 59.408208354864904,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 0.0,
        "score": 88.67758673706858
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 59.408208354864904
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 65.352863197908,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 77.72525626039064
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 65.352863197908
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 84.55664975223662,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100.0,
        "score": 66.51791499546789
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 84.55664975223662
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 63.0110398677102,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 11.255341825352849
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.0110398677102
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 48.74742459938019,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 100.0,
        "score": 75.27453186265669
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 48.74742459938019
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 67.9651992139091,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 83.89559764172728
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 67.9651992139091
      },
      "model_routing": {
       "accuracy": 0.8894009216589862,
       "cc": 75.02535719232871,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 87.87878787878789,
        "score": 44.21027071118523
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.02535719232871
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 31.98964337733368,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -100.0,
        "score": 95.96893013200102
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 31.98964337733368
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 54.772780391389254,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 30.985007840834434
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.772780391389254
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 50.878022896636004,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 52.634068689908005
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 50.878022896636004
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 43.5106719476208,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": -33.333333333333336,
        "score": 78.35810279938414
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 43.5106719476208
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.85458175009258,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.56374525027778
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.85458175009258
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.30671773822844,
      91.70031153434485
     ],
     "I": [
      61.6000039357046,
      71.43891205182712
     ],
     "capability": [
      75.29188998324385,
      80.61305668502236
     ],
     "composite": {
      "A": [
       10.629002654265207,
       10.928544162470537
      ],
      "B": [
       11.114542762869075,
       11.645757487636661
      ],
      "C": [
       10.629002654265207,
       10.928544162470537
      ]
     },
     "seed": 72
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.90819924197592,
     "capability": 78.8040995813651,
     "intelligence": 66.69999992075427
    },
    "per_type": {
     "choice": {
      "accuracy": 0.9186666666666666,
      "calibration": 91.8112970311977,
      "cc": 89.49463250349223,
      "cc_by_type": {
       "choice": 89.49463250349223
      },
      "cc_tiered": 91.43468725529016,
      "cc_tiered_open": 94.7183214036337,
      "cc_tiered_sealed": 88.15105310694662,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 89.49463250349223
     },
     "noul": {
      "accuracy": 0.744,
      "calibration": 89.64449693220469,
      "cc": 48.8,
      "cc_by_type": {
       "noul": 48.8
      },
      "cc_tiered": 47.150637613782905,
      "cc_tiered_open": 47.46790890269151,
      "cc_tiered_sealed": 46.833366324874305,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 48.8
     },
     "score": {
      "accuracy": 0.7973333333333333,
      "calibration": 91.2688037625254,
      "cc": 62.42466072114584,
      "cc_by_type": {
       "score": 62.42466072114584
      },
      "cc_tiered": 61.514674893189735,
      "cc_tiered_open": 61.328583803232945,
      "cc_tiered_sealed": 61.700765983146525,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.42466072114584
     }
    },
    "raw_on_A": {
     "calibration": 89.10291663559049,
     "capability": 78.82003448345057,
     "intelligence": 68.53715233131064
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.026666666666666616,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.372720190039681,
    "Emphasis on Accuracy 60:20:20": 10.974466792343826,
    "Emphasis on Cost 20:20:60": 7.097898721775836,
    "Emphasis on Speed 20:60:20": 11.354543720931087,
    "Intelligence only": 14.757406959769908
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "StartLux Labs",
   "axes": {
    "calibration": 91.92111224732803,
    "cost": 30.41280058663709,
    "intelligence": 73.19432958384127,
    "speed": 90.32360333729886
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.040930804934676546,
      "ece_score": 91.8138390130647,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.8138390130647,
      "score_before_time_limit": 91.8138390130647,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.17962466487935658,
      "brier": 0.04448579796465505,
      "ece": 0.04709348102978012,
      "n": 373,
      "score": 90.58130379404398,
      "score_before_time_limit": 90.58130379404398,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9596774193548387,
      "interval90_width": 0.20206093189964155,
      "n": 372,
      "nrps": 0.04615661957496166,
      "score": 93.3681939348754,
      "score_before_time_limit": 93.3681939348754,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04323975086376515
     }
    },
    "score": 91.92111224732803
   },
   "capability": 82.55772091558464,
   "composite_ci95": {
    "A": [
     21.233879938457505,
     21.79850561773589
    ],
    "B": [
     21.963666749450166,
     22.858083929840905
    ],
    "C": [
     21.233879938457505,
     21.79850561773589
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module: base Qwen/Qwen3.8-27B -> qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output. Estimated, not charged, not a quote from the author or any host. Frozen pricing_v15: base-model references unchanged since sha256 3948e3f1...ea4d0, the module this row was first priced with (the later re-freeze only adds a hosted-API launch-list key), USD per 1,000 decisions over the common 1,485-item cost basis (the 15 very long items are left out of cost for every row; refused items cost nothing and still count as wrong answers). The model cards do not name the Qwen version; Qwen/Qwen3.8-27B is the base the Decision Index metadata gives for this model, consistent with the NOTICE (modified Alibaba Cloud Apache-2.0 model) and with the config (Qwen3_5ForConditionalGeneration, 64 layers, hidden 5120), the same reference as the live torchcast-decision-27b row (a fine-tune of StartLux-Decision-27B v1). OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.208724 per 1,000 decisions on the common basis (USD 0.206637 over all 1,500 items) is 3.231x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.208724 per 1,000 decisions (3.231x the official cost cap USD 0.064595), against USD 0.206637 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.20663720000000002
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.20872444444444446
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "StartLux-Decision v1.1 27B Instant (StartLux Labs, Qwen3.8-27B fine-tune)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H200 141 GB (sm_90, driver 580.173.02), Finland (EU). The author's own server on vLLM 0.19.1, offline, loopback, one request at a time.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 77.73279465771967,
    "I_sealed": 68.65586450996287,
    "base": 73.19432958384127,
    "excess": -0.7625966225989131,
    "gap": 9.076930147756798,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 87.38989648625872,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 88.67568417741427,
       "judge": 88.13447593935399,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 72.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 72.86024732126884,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.15270644756282,
       "hard": 80.70289974380364,
       "judge": 72.2051509514639,
       "standard": 58.01135746775971
      }
     },
     "sealed|choice": {
      "cc": 83.43695541516112,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 74.49982652943217,
       "judge": 87.65915768854063,
       "standard": 90.50833157561695
      }
     },
     "sealed|noul": {
      "cc": 57.713563842230855,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 49.09090909090909,
       "judge": 60.00000000000001,
       "standard": 64.17910447761194
      }
     },
     "sealed|score": {
      "cc": 64.81707427249664,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 87.88715519480432,
       "hard": 62.862170937119835,
       "judge": 62.78531836211132,
       "standard": 60.23947434767438
      }
     }
    }
   },
   "jevbench_score": 21.60649939785744,
   "key": "startlux-decision-v1.1-27b-instant",
   "last_measured_on": "2026-10-16",
   "licence": "Weights CC BY-NC 4.0 (non-commercial: commercial use needs a separate licence from StartLux Labs; listing a measured score is evaluation use); inference package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "startlux-models/StartLux-Decision-v1.1-27B-Instant@32e7d35d74db615c4a059181efbf21c5421f3561",
   "noul_decisive": {
    "acc_among_decisive": 0.9934640522875817,
    "decisive": 306,
    "decisive_rate": 0.8203753351206434,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.92111224732803,
     "cost": 30.41280058663709,
     "intelligence": 73.19432958384127,
     "speed": 90.62312905261143
    },
    "board_release": "v1.6.43",
    "capability": 82.55772091558464,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      21.245034378926533,
      21.810261320192286
     ],
     "B": [
      21.973213364261966,
      22.86842408467527
     ],
     "C": [
      21.245034378926533,
      21.810261320192286
     ]
    },
    "jevbench_score": 21.618048863688184,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": 85,
    "rank": 118,
    "ranks": {
     "A": 118,
     "B": 114,
     "C": 101,
     "capability": 14
    },
    "scores": {
     "A": 21.618048863688184,
     "B": 22.526778205217695,
     "C": 21.618048863688184
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2602560680010356,
     "p50_s_raw": 0.0551280340005178,
     "p95_s_adjusted": 0.33287887162878177,
     "p95_s_raw": 0.09143943581439089
    },
    "views": {
     "Balanced 33:33:33": 19.27695949442568,
     "Emphasis on Accuracy 60:20:20": 21.788279630595614,
     "Emphasis on Cost 20:20:60": 14.99823406009409,
     "Emphasis on Speed 20:60:20": 23.225862696351562,
     "Intelligence only": 27.08009959487104
    }
   },
   "rank": 134,
   "ranked": true,
   "ranks": {
    "A": 134,
    "B": 129,
    "C": 116,
    "capability": 16
   },
   "repo": "https://huggingface.co/startlux-models/StartLux-Decision-v1.1-27B-Instant",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.60649939785744,
    "B": 22.516744631015868,
    "C": 21.60649939785744
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26417634401004764,
    "p50_s_raw": 0.05708817200502381,
    "p95_s_adjusted": 0.3513546783942729,
    "p95_s_raw": 0.10067733919713646
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 94.60413033269491,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 92.14572433141801
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 94.60413033269491
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 54.896811745749496,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": 43.99999999999999,
        "score": 65.22914149387627
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 54.896811745749496
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 84.58165491823706,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 85.18518518518519,
        "score": 78.3158771305016
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 84.58165491823706
      },
      "fairness": {
       "accuracy": 0.9166666666666666,
       "cc": 84.39784298036983,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 73.33333333333334,
        "score": 98.90781465539521
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.39784298036983
      },
      "jaggedness": {
       "accuracy": 0.6777777777777778,
       "cc": 49.005904102153835,
       "cc_by_type": {
        "choice": 68.23869104908566,
        "noul": 18.918918918918926,
        "score": 59.86010233845691
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.005904102153835
      },
      "llm_router": {
       "accuracy": 0.9456869009584664,
       "cc": 84.73240889274621,
       "cc_by_type": {
        "choice": 97.62261160517741,
        "noul": 95.8762886597938,
        "score": 60.698326413267424
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.73240889274621
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 58.29924872452923,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 50.0,
        "score": 62.01114823544332
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 58.29924872452923
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 69.69226325427245,
       "cc_by_type": {
        "choice": 87.28399087055756,
        "noul": 50.0,
        "score": 71.79279889225978
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.69226325427245
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 72.38166965157417,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 57.1450089547225
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 72.38166965157417
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 80.83683405690114,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 57.525517185718435
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.83683405690114
      },
      "usecase": {
       "accuracy": 0.8227272727272728,
       "cc": 65.68990894056331,
       "cc_by_type": {
        "choice": 87.74509803921569,
        "noul": 37.5,
        "score": 71.82462878247424
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 65.68990894056331
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 60.96108872191072,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100.0,
        "score": 20.978504260970265
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 60.96108872191072
      },
      "cs": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 100.0
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 61.30656762125532,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 50.58636953043263
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 61.30656762125532
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.47138592260931,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 83.30796307756245
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.47138592260931
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.71241850166923,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.99014191096676
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.71241850166923
      },
      "en": {
       "accuracy": 0.8331885317115552,
       "cc": 71.69241929432023,
       "cc_by_type": {
        "choice": 84.18374196067901,
        "noul": 63.309352517985616,
        "score": 67.58416340429606
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.69241929432023
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 77.55106151830554,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 32.653184554916635
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 77.55106151830554
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.7312947809802,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.86055100960733
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.7312947809802
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 78.71872583341194,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100.0,
        "score": 99.68558926494168
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 78.71872583341194
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 82.14587496948548,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75.0,
        "score": 88.19740144476927
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 82.14587496948548
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 1.0,
       "cc": 77.6224564529621,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 32.867369358886286
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 77.6224564529621
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 50.59151729695487,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 31.774551890864622
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 50.59151729695487
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 12.928731135718815,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 38.78619340715645
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 12.928731135718815
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.62243022913351,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.53395735406725
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.62243022913351
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 29.967630712302093,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 23.236225470239614
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 29.967630712302093
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 75.83899608734215,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 75.13442372542049
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 75.83899608734215
      },
      "pt": {
       "accuracy": 0.8928571428571429,
       "cc": 85.66738033499985,
       "cc_by_type": {
        "choice": 89.39304655274012,
        "noul": 100.0,
        "score": 67.6090944522594
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 85.66738033499985
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 61.56852305958395,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 50.23748407236888
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 61.56852305958395
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 56.86499088250887,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 70.5949726475266
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 56.86499088250887
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.31956327674492,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 99.47481886249281
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.31956327674492
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 57.8305341335463,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 40.158269067305575
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 57.8305341335463
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8813333333333333,
       "cc": 84.67249660345587,
       "cc_by_type": {
        "choice": 84.67249660345587
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.67249660345587
      },
      "noul": {
       "accuracy": 0.8106666666666666,
       "cc": 62.133333333333326,
       "cc_by_type": {
        "noul": 62.133333333333326
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.133333333333326
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 67.32764882650292,
       "cc_by_type": {
        "score": 67.32764882650292
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.32764882650292
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 29.881576050153324,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -25.0,
        "score": 46.64472815045997
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 29.881576050153324
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 87.04724612514936,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 79.77527874811891
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 87.04724612514936
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 56.37200356001129,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 67.89649848491193
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 56.37200356001129
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 1.3223643717638087,
       "cc_by_type": {
        "choice": 64.91228070175438,
        "noul": -33.333333333333336,
        "score": -27.61185425312962
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 1.3223643717638087
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 46.99599177928393,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 88.81406229437354
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 46.99599177928393
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 45.49259456532935,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 15.723066714855982
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 45.49259456532935
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 62.43686377152064,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 14.28571428571428,
        "score": 73.02487702884764
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 62.43686377152064
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 48.71896369085224,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 0.0,
        "score": 67.06281441750444
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 48.71896369085224
      },
      "insurance_claims": {
       "accuracy": 0.8,
       "cc": 63.77044581469289,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 71.31133744407869
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 63.77044581469289
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 77.83384735706747,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 66.83487540453574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 77.83384735706747
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 65.3802683789926,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 52.21315655816645
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 65.3802683789926
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 41.376624819304425,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 55.31267015683801
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 41.376624819304425
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 63.70402277234668,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 71.11206831704003
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 63.70402277234668
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 79.45253805021827,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 93.93939393939394,
        "score": 49.09354488658552
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.45253805021827
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 59.55706008248318,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0.0,
        "score": 90.18749502672017
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 59.55706008248318
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 88.75474237649098,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 66.26422712947296
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 88.75474237649098
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 45.51020954722372,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 51.30895376482386
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 45.51020954722372
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 50.702961440976175,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 0.0,
        "score": 66.60163794611695
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 50.702961440976175
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.97000103064437,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.91000309193309
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.97000103064437
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.15955420517918,
      92.63070738800778
     ],
     "I": [
      68.6761427354401,
      76.80063493292705
     ],
     "capability": [
      78.99248244635237,
      84.23659103542126
     ],
     "composite": {
      "A": [
       21.233879938457505,
       21.79850561773589
      ],
      "B": [
       21.963666749450166,
       22.858083929840905
      ],
      "C": [
       21.233879938457505,
       21.79850561773589
      ]
     },
     "seed": 73
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.92111224732803,
     "capability": 82.55772091558464,
     "intelligence": 73.19432958384127
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8813333333333333,
      "calibration": 91.8138390130647,
      "cc": 84.67249660345587,
      "cc_by_type": {
       "choice": 84.67249660345587
      },
      "cc_tiered": 85.41342595070992,
      "cc_tiered_open": 87.38989648625872,
      "cc_tiered_sealed": 83.43695541516112,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.67249660345587
     },
     "noul": {
      "accuracy": 0.8106666666666666,
      "calibration": 90.58130379404398,
      "cc": 62.133333333333326,
      "cc_by_type": {
       "noul": 62.133333333333326
      },
      "cc_tiered": 65.33090200393116,
      "cc_tiered_open": 72.94824016563147,
      "cc_tiered_sealed": 57.713563842230855,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.133333333333326
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 93.3681939348754,
      "cc": 67.32764882650292,
      "cc_by_type": {
       "score": 67.32764882650292
      },
      "cc_tiered": 68.83866079688275,
      "cc_tiered_open": 72.86024732126884,
      "cc_tiered_sealed": 64.81707427249664,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.32764882650292
     }
    },
    "raw_on_A": {
     "calibration": 88.14525609544745,
     "capability": 79.34618044792921,
     "intelligence": 70.54710480041096
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.264716127093127,
    "Emphasis on Accuracy 60:20:20": 21.77889300008525,
    "Emphasis on Cost 20:20:60": 14.99378567469301,
    "Emphasis on Speed 20:60:20": 23.193894526925927,
    "Intelligence only": 27.08009959487104
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SubQ",
   "axes": {
    "calibration": 88.06513871774217,
    "cost": 60.67742028447466,
    "intelligence": 75.75710514449688,
    "speed": 84.90104208468355
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05760546714540509,
      "ece_score": 88.47890657091898,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.47890657091898,
      "score_before_time_limit": 88.47890657091898,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.11199999999999999,
      "brier": 0.07228155825255182,
      "ece": 0.07309512757394987,
      "n": 375,
      "score": 85.38097448521003,
      "score_before_time_limit": 85.38097448521003,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.944,
      "interval90_width": 0.16422222222222224,
      "n": 375,
      "nrps": 0.049061246713496494,
      "score": 90.33553509709749,
      "score_before_time_limit": 90.33553509709749,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07211402567227683
     }
    },
    "score": 88.06513871774217
   },
   "capability": 81.91112193111952,
   "composite_ci95": {
    "A": [
     74.09061953247686,
     76.6644175462098
    ],
    "B": [
     73.52994358661283,
     77.08716692169163
    ],
    "C": [
     74.09061953247686,
     76.6644175462098
    ]
   },
   "cost": {
    "basis": "Vendor's published standard launch tariff, USD 0.04 per million input tokens; output free (read 22 Oct 2026 at https://docs.subq.ai/using-the-api/models/ and https://subq.ai). Counted under the frozen I-1 launch-list-price rule. Capacity-limited public preview, first come first served, with a waitlist once capacity is reached; not openly bookable by everyone. Promotional signup credits are excluded. Own measured usage: 1,939,205 input / 0 output tokens over all 1,500 decisions. Common 1,485-item cost basis: 759,310 input / 0 output tokens, USD 0.020452795 per 1,000 decisions; all-item cost USD 0.051712133 per 1,000. The historical preview row keeps its frozen USD 0.05/M input tariff.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.051712133333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020452794612794613
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SubQ Snap 1.1 (SubQ)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.subq.ai /v1/systemone, served model subq-snap-1.1, measured 22 Oct 2026 from the evaluator host, official per-item latency subset without API adjustment. Full S1200+P300 draw; single same-author second use, provider saw the sealed items in the 13 Oct preview run. Native probabilities, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Inputs retained indefinitely by default; this run used already-retired items.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 81.19768798375172,
    "I_sealed": 70.31652230524202,
    "base": 75.75710514449688,
    "excess": 1.0416389081539918,
    "gap": 10.881165678509703,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 88.50761395257393,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 88.67568417741427,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 83.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 70.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 72.0081553126909,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.478885212283465,
       "hard": 75.63135702400898,
       "judge": 80.3971244828602,
       "standard": 56.442933185004485
      }
     },
     "sealed|choice": {
      "cc": 81.88359847741832,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 79.70394356424192,
       "judge": 80.60724779627816,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 61.722683903847056,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 52.727272727272734,
       "judge": 63.07692307692307,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 67.34328453446071,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 82.75405644034946,
       "hard": 69.4360268544149,
       "judge": 62.2790519956979,
       "standard": 63.048762749752164
      }
     }
    }
   },
   "jevbench_score": 75.73702535215112,
   "key": "subq-snap-1.1",
   "last_measured_on": "2026-10-22",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.43",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://subq.ai#subq-snap-1.1",
   "noul_decisive": {
    "acc_among_decisive": 0.9459459459459459,
    "decisive": 333,
    "decisive_rate": 0.888,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 4,
    "axes": {
     "calibration": 88.06513871774217,
     "cost": 60.67742028447466,
     "intelligence": 75.75710514449688,
     "speed": 84.94774043787051
    },
    "board_release": "v1.6.43",
    "capability": 81.91112193111952,
    "capability_api_board_rank": 4,
    "capability_open_board_rank": null,
    "capability_rank": 7,
    "composite_ci95": {
     "A": [
      74.09950653419286,
      76.6739327545555
     ],
     "B": [
      73.53694581918141,
      77.09486308430787
     ],
     "C": [
      74.09950653419286,
      76.6739327545555
     ]
    },
    "jevbench_score": 75.7463117322681,
    "measured_in": "v1.6.43",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 5,
    "ranks": {
     "A": 5,
     "B": 4,
     "C": 5,
     "capability": 16
    },
    "scores": {
     "A": 75.7463117322681,
     "B": 75.74847016866207,
     "C": 75.7463117322681
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4687188219977543,
     "p50_s_raw": 0.4687188219977543,
     "p95_s_adjusted": 0.682831523841014,
     "p95_s_raw": 0.682831523841014
    },
    "views": {
     "Balanced 33:33:33": 72.3717813701936,
     "Emphasis on Accuracy 60:20:20": 73.68894327115758,
     "Emphasis on Cost 20:20:60": 67.19182812608965,
     "Emphasis on Speed 20:60:20": 76.92721011051661,
     "Intelligence only": 75.75710514449688
    }
   },
   "rank": 6,
   "ranked": true,
   "ranks": {
    "A": 6,
    "B": 5,
    "C": 6,
    "capability": 19
   },
   "repo": "https://subq.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 75.73702535215112,
    "B": 75.74104045901554,
    "C": 75.73702535215112
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4690868764882907,
    "p50_s_raw": 0.4690868764882907,
    "p95_s_adjusted": 0.6896718636329751,
    "p95_s_raw": 0.6896718636329751
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Hosted SubQ Snap 1.1; measured 22 Oct 2026. SubQ Snap 1.1 is a capacity-limited public preview (first come first served, with a waitlist). SAME-AUTHOR SECOND use of this draw: SubQ already saw the sealed items on 13 Oct 2026 through subq-snap-preview; version 1.1 was released afterwards. The old preview name now routes to the same model; its historical row remains labelled as the 13 Oct measurement, with its frozen USD 0.05/M tariff unchanged. This row uses the published USD 0.04/M input tariff, output free, excluding signup credits. All 1,500 items answered once with native probabilities and reported token usage. Permission-error requests never reached the model; collection resumed after access was restored. S/P check versus the preview: sealed accuracy 84.25% vs 85.00% (-0.75 pp), public 90.33% vs 88.67% (+1.67 pp); sealed-minus-public gain -2.42 pp, 95% CI [-6.00, +1.17] pp, zero inside: not flagged. No evidence of public-item tuning in this check; prior exposure remains disclosed.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 85.59462369626596,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 65.11720442213118
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 85.59462369626596
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 61.34400703600102,
       "cc_by_type": {
        "choice": 68.18663838812301,
        "noul": 43.99999999999999,
        "score": 71.84538271988004
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.34400703600102
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 90.32923568776323,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 100.0,
        "score": 77.4917721039401
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.32923568776323
      },
      "fairness": {
       "accuracy": 0.8958333333333334,
       "cc": 83.27632603982205,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 89.82897811946613
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.27632603982205
      },
      "jaggedness": {
       "accuracy": 0.7333333333333333,
       "cc": 59.79508259236162,
       "cc_by_type": {
        "choice": 65.35129932627527,
        "noul": 40.54054054054055,
        "score": 73.49340791026906
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 59.79508259236162
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 83.1733277544028,
       "cc_by_type": {
        "choice": 92.86783481553225,
        "noul": 97.93814432989691,
        "score": 58.714004117779254
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.1733277544028
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": -8.212860511185264,
       "cc_by_type": {
        "choice": 48.387096774193544,
        "noul": -100.0,
        "score": 26.97432169225067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 45.07843930656005,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 0.0,
        "score": 59.977585960917246
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 45.07843930656005
      },
      "multilingual": {
       "accuracy": 0.8109090909090909,
       "cc": 68.1315293010611,
       "cc_by_type": {
        "choice": 82.3932181284643,
        "noul": 55.55555555555556,
        "score": 66.44581421916344
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.1315293010611
      },
      "negation_pair_gen": {
       "accuracy": 0.9069767441860465,
       "cc": 78.40427254862233,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 68.54615097920032
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.40427254862233
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 77.85381509880871,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100.0,
        "score": 51.57946331444412
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 77.85381509880871
      },
      "usecase": {
       "accuracy": 0.8090909090909091,
       "cc": 67.41426811771518,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 54.16666666666667,
        "score": 68.9094710198122
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 67.41426811771518
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 76.87644537854416,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 49.676955183251536
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 76.87644537854416
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 52.01247549696391,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 43.44142034649848
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.01247549696391
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 91.23772581977327,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100.0,
        "score": 81.67777922923129
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 91.23772581977327
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.71362792645883,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.9937701853355
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.71362792645883
      },
      "en": {
       "accuracy": 0.8462206776715899,
       "cc": 75.36424883571205,
       "cc_by_type": {
        "choice": 83.52473120904064,
        "noul": 71.22302158273382,
        "score": 71.34499371536167
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.36424883571205
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 74.6441100390774,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 23.932330117232226
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 74.6441100390774
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.54067102595756,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.075564990441
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.54067102595756
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.32987213230238,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 95.34255757337776
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.32987213230238
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 47.270696258480406,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 0.0,
        "score": 41.812088775441225
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 47.270696258480406
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.8886522963651,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -3.304425028285829
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.8886522963651
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 38.90739241902529,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 39.99999999999999,
        "score": -8.980761583273612
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.90739241902529
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 24.116527114148045,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 72.34958134244413
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 24.116527114148045
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.61321393008984,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.50630845693618
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.61321393008984
      },
      "nl": {
       "accuracy": 0.7,
       "cc": -2.0136452061382903,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -72.70760228508153
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8974358974358975,
       "cc": 82.55541849674057,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80.0,
        "score": 75.28369095361572
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 82.55541849674057
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 76.47339934882655,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 61.24105838825926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.47339934882655
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 67.90107670920433,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 69.23514502122998
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 67.90107670920433
      },
      "tr": {
       "accuracy": 1.0,
       "cc": 94.95103283164401,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 84.85309849493206
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 94.95103283164401
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.94915988090104,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 95.36360867496114
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.94915988090104
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 47.08691030844218,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 7.927397591993202
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 47.08691030844218
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8733333333333333,
       "cc": 83.63918176773377,
       "cc_by_type": {
        "choice": 83.63918176773377
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.63918176773377
      },
      "noul": {
       "accuracy": 0.84,
       "cc": 68.0,
       "cc_by_type": {
        "noul": 68.0
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 68.0
      },
      "score": {
       "accuracy": 0.7653333333333333,
       "cc": 68.99136919378957,
       "cc_by_type": {
        "score": 68.99136919378957
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 68.99136919378957
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 37.1147079515983,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 0.0,
        "score": 27.3441238547949
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 37.1147079515983
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 79.64986035349233,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 61.1718032826992
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 79.64986035349233
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 66.10739036282742,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 30.842496291734278
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 66.10739036282742
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -17.615704748395572,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -116.00500898202881
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.66475542632195,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 74.64644019200932
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.66475542632195
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 59.13377530019597,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": 69.22522527165711
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 59.13377530019597
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 57.10562696146352,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 66.83508816730375
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.10562696146352
      },
      "gaming": {
       "accuracy": 0.9090909090909091,
       "cc": 77.68713959818707,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50.0,
        "score": 93.51438046703507
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 77.68713959818707
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 17.776309130734905,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60.0,
        "score": 34.16226072553804
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 17.776309130734905
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 82.68459256985513,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 81.38711104289872
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 82.68459256985513
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 58.988451873179145,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 55.25992926294832
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 58.988451873179145
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 27.71488816719786,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 46.58552471664734
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 27.71488816719786
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 91.33913402560951,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 74.01740207682855
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 91.33913402560951
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 79.3376077944341,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 96.96969696969697,
        "score": 44.54961992009885
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.3376077944341
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 44.770304956543136,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0.0,
        "score": 68.85985920744133
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 44.770304956543136
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 63.17987181140378,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 72.91937402288359
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 63.17987181140378
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 38.55486178559623,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": 45.2212356030941
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 38.55486178559623
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 46.64142601531304,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": 33.33333333333333,
        "score": 79.0547128285478
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 46.64142601531304
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.814456390198,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.44336917059398
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.814456390198
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.27453802496953,
      90.10715200144764
     ],
     "I": [
      71.22059116474675,
      78.89225696940721
     ],
     "capability": [
      78.27586653706702,
      83.99171809392114
     ],
     "composite": {
      "A": [
       74.09061953247686,
       76.6644175462098
      ],
      "B": [
       73.52994358661283,
       77.08716692169163
      ],
      "C": [
       74.09061953247686,
       76.6644175462098
      ]
     },
     "seed": 78
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.06513871774217,
     "capability": 81.91112193111952,
     "intelligence": 75.75710514449688
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8733333333333333,
      "calibration": 88.47890657091898,
      "cc": 83.63918176773377,
      "cc_by_type": {
       "choice": 83.63918176773377
      },
      "cc_tiered": 85.19560621499613,
      "cc_tiered_open": 88.50761395257393,
      "cc_tiered_sealed": 81.88359847741832,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.63918176773377
     },
     "noul": {
      "accuracy": 0.84,
      "calibration": 85.38097448521003,
      "cc": 68.0,
      "cc_by_type": {
       "noul": 68.0
      },
      "cc_tiered": 72.3999892949187,
      "cc_tiered_open": 83.07729468599035,
      "cc_tiered_sealed": 61.722683903847056,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 68.0
     },
     "score": {
      "accuracy": 0.7653333333333333,
      "calibration": 90.33553509709749,
      "cc": 68.99136919378957,
      "cc_by_type": {
       "score": 68.99136919378957
      },
      "cc_tiered": 69.6757199235758,
      "cc_tiered_open": 72.0081553126909,
      "cc_tiered_sealed": 67.34328453446071,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.99136919378957
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.36047856736685,
    "Emphasis on Accuracy 60:20:20": 73.68191206349243,
    "Emphasis on Cost 20:20:60": 67.18598208536949,
    "Emphasis on Speed 20:60:20": 76.90422650524084,
    "Intelligence only": 75.75710514449688
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SubQ",
   "axes": {
    "calibration": 90.69740052356492,
    "cost": 57.693900899055016,
    "intelligence": 72.25005247787018,
    "speed": 86.35826133612204
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04418888669165353,
      "ece_score": 91.16222266166929,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 91.16222266166929,
      "score_before_time_limit": 91.16222266166929,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.14400000000000002,
      "brier": 0.07100960878955642,
      "ece": 0.045594591092318275,
      "n": 375,
      "score": 90.88108178153634,
      "score_before_time_limit": 90.88108178153634,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9653333333333334,
      "interval90_width": 0.22466666666666665,
      "n": 375,
      "nrps": 0.047718109524649006,
      "score": 90.04889712748917,
      "score_before_time_limit": 90.04889712748917,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.0756519739627838
     }
    },
    "score": 90.69740052356492
   },
   "capability": 81.47372650071756,
   "composite_ci95": {
    "A": [
     72.52502278899739,
     75.25211552563907
    ],
    "B": [
     71.43810693082546,
     75.36008043389832
    ],
    "C": [
     72.52502278899739,
     75.25211552563907
    ]
   },
   "cost": {
    "basis": "Operator's own published standard usage tariff: USD 0.05 per 1M INPUT tokens with no output billing. Two vendor sources, both snapshotted: the pricing section of https://subq.ai/ read on 14 Oct 2026 08:10Z ('In $0.05 per 1M tokens Input / Out Free Output') and the vendor's own launch post 'Introducing SubQ Snap' ('SubQ Snap is priced at $0.05 per million input tokens, with no charge for output tokens'), which also dates the tariff to the product's launch; Subquadratic's Alexander Whedon separately confirmed the figure in writing to Florian on 14 Oct 2026 01:16 Berlin (board #19133). Counted under interpretation I-1 (a manufacturer's standard, non-promotional launch list price counts from day 1 - the reading that already admitted kushal-gemma4-31b-it-autoloops). DISCLOSED: Snap is in a capacity-limited public preview - first come, first served, with a waitlist once capacity is reached - so this is the vendor's stated usage price for a product that is not yet openly bookable by everyone; CoS deputy decision #19174 rules explicitly that I-1 covers this tariff during that public preview, on the condition that the capacity limit and waitlist are stated here, as they are. The same pricing section offers 'Free 100M tokens for the first 10,000 users': that is a promotional trial credit and is deliberately NOT used - a free tier is priced at the published paid tariff, never as cost 0. The tariff is multiplied by this row's OWN measured input tokens (1,943,705 over 1,500 answered rows, mean 1,296) and divided by the 1,485 items of the common cost basis the scorer uses. The output term is zero twice over: the tariff bills no output, and the vendor's own usage meter reports 0 output tokens on all 1,500 rows. Gate chain: CoS deputy #18975 made the cost axis conditional on a separate frozen-pricing change passing re-freeze, unchanged-row proof and independent review; the chief of staff asked for it in #19133; the scoped change adds this one key to pricing_v15.LAUNCH_LIST_PRICE_API and nothing else (r113/pricing/CANDIDATE.diff; unchanged-row proof r113/receipts/PRICING-EQUIV-R113.json and two rounds of independent review, r113/review-pricing/REVIEW.json (REVISE) and r113/review-pricing-r2/REVIEW.json (PASS); the findings about this row's text and the builder's gates are applied here, the two about the publication layer are on the build sheet). INSIDE THE OFFICIAL COST CAP on the common basis: USD 0.025716 per 1,000 decisions is 0.398x the official cap USD 0.064595. Over all 1,500 items the row cost USD 0.064790 (1.003x the cap) and was disclosed as ranked over the cap in v1.6.29; that disclosure is superseded. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.025716 per 1,000 decisions (0.398x the official cost cap USD 0.064595), against USD 0.064790 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.06479016666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025715993265993265
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SubQ Snap (preview, SubQ)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.subq.ai /v1/systemone API, model `subq-snap-preview`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `subq` before the first request (CoS deputy #18975; every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Florian's work order (X, 12 Oct 2026). SubQ retains API inputs indefinitely by default; acceptable only because this draw's items are already retired.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 75.21805946266113,
    "I_sealed": 69.28204549307924,
    "base": 72.25005247787018,
    "excess": -3.903512800773818,
    "gap": 5.936013969581893,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 84.9873634422428,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 83.01352626612143,
       "judge": 80.22412656558998,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 73.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 67.54604199888068,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 64.92519654954472,
       "hard": 76.69157031955869,
       "judge": 72.3442858248807,
       "standard": 43.36804234319263
      }
     },
     "sealed|choice": {
      "cc": 84.72260612932492,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 78.66312015727996,
       "judge": 87.65915768854063,
       "standard": 88.60999789074035
      }
     },
     "sealed|noul": {
      "cc": 57.554016750105276,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 49.09090909090909,
       "judge": 60.00000000000001,
       "standard": 58.20895522388059
      }
     },
     "sealed|score": {
      "cc": 65.56951359980751,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 85.17961744525421,
       "hard": 67.30197547341396,
       "judge": 61.73385013050089,
       "standard": 58.05303313383112
      }
     }
    }
   },
   "jevbench_score": 74.37845073677168,
   "key": "subq-snap-preview",
   "last_measured_on": "2026-10-13",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.subq.ai/v1/systemone#subq-snap-preview",
   "noul_decisive": {
    "acc_among_decisive": 0.9532710280373832,
    "decisive": 321,
    "decisive_rate": 0.856,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 7,
    "axes": {
     "calibration": 90.69740052356492,
     "cost": 57.693900899055016,
     "intelligence": 72.25005247787018,
     "speed": 86.40460211903752
    },
    "board_release": "v1.6.43",
    "capability": 81.47372650071756,
    "capability_api_board_rank": 5,
    "capability_open_board_rank": null,
    "capability_rank": 10,
    "composite_ci95": {
     "A": [
      72.53319025366382,
      75.26090880321355
     ],
     "B": [
      71.44444637250206,
      75.36713509179089
     ],
     "C": [
      72.53319025366382,
      75.26090880321355
     ]
    },
    "jevbench_score": 74.38704101088614,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 8,
    "ranks": {
     "A": 8,
     "B": 8,
     "C": 8,
     "capability": 21
    },
    "scores": {
     "A": 74.38704101088614,
     "B": 73.94958963293669,
     "C": 74.38704101088614
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.46557116050098557,
     "p50_s_raw": 0.46557116050098557,
     "p95_s_adjusted": 0.4915341741005249,
     "p95_s_raw": 0.4915341741005249
    },
    "views": {
     "Balanced 33:33:33": 70.18014569964379,
     "Emphasis on Accuracy 60:20:20": 70.99371113720164,
     "Emphasis on Cost 20:20:60": 64.58876677881489,
     "Emphasis on Speed 20:60:20": 75.87938757645203,
     "Intelligence only": 72.25005247787018
    }
   },
   "rank": 10,
   "ranked": true,
   "ranks": {
    "A": 10,
    "B": 10,
    "C": 10,
    "capability": 24
   },
   "repo": "https://subq.ai",
   "repriced_in": "v1.6.20",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.37845073677168,
    "B": 73.94279784299546,
    "C": 74.37845073677168
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4660752070049057,
    "p50_s_raw": 0.4660752070049057,
    "p95_s_adjusted": 0.49626982034533285,
    "p95_s_raw": 0.49626982034533285
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 89.67339746240113,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 77.35352572053671
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 89.67339746240113
      },
      "calibration_gen": {
       "accuracy": 0.7524752475247525,
       "cc": 59.325989058477944,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 36.00000000000001,
        "score": 70.60999262612313
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 59.325989058477944
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 89.73374806927508,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 100.0,
        "score": 75.70530924847564
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.73374806927508
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.11997769875624,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 95.35993309626873
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.11997769875624
      },
      "jaggedness": {
       "accuracy": 0.7833333333333333,
       "cc": 60.15465710190771,
       "cc_by_type": {
        "choice": 78.34456207892205,
        "noul": 29.729729729729737,
        "score": 72.38967949707134
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 60.15465710190771
      },
      "llm_router": {
       "accuracy": 0.9073482428115016,
       "cc": 80.88165044070985,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 95.8762886597938,
        "score": 56.278216241626076
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.88165044070985
      },
      "long_state": {
       "accuracy": 0.4666666666666667,
       "cc": -25.521900617988198,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -12.049572821706533
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 33.822823442121184,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": -25.0,
        "score": 51.21073836760064
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 33.822823442121184
      },
      "multilingual": {
       "accuracy": 0.7745454545454545,
       "cc": 63.63301054731818,
       "cc_by_type": {
        "choice": 79.45875448320835,
        "noul": 50.0,
        "score": 61.44027715874617
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.63301054731818
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 77.29805491838812,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 73.33333333333334,
        "score": 65.22749808849767
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 77.29805491838812
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 84.09170943563231,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 64.28714031890895
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.09170943563231
      },
      "usecase": {
       "accuracy": 0.7909090909090909,
       "cc": 64.3360385766533,
       "cc_by_type": {
        "choice": 85.29411764705883,
        "noul": 41.66666666666667,
        "score": 66.04733141623439
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.3360385766533
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 61.46987986833912,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100.0,
        "score": 22.504877700255456
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 61.46987986833912
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 42.044641118681675,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 13.537917211651784
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 42.044641118681675
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.30873412452327,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 76.82000768330435
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.30873412452327
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.17191426416157,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 98.7019625317771
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.17191426416157
      },
      "en": {
       "accuracy": 0.8523023457862728,
       "cc": 74.39868278842214,
       "cc_by_type": {
        "choice": 85.94110396504801,
        "noul": 66.90647482014388,
        "score": 70.34846958007451
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.39868278842214
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 68.10721678264782,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 14.10957856980316
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.10721678264782
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.95984983965347,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 97.33310143152872
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.95984983965347
      },
      "fr": {
       "accuracy": 1.0,
       "cc": 98.39208642380281,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.17625927140838
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 98.39208642380281
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 51.02919425601916,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 0.0,
        "score": 53.08758276805747
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 51.02919425601916
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 67.3748258425729,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 16.15409561033757
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.3748258425729
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 39.17673354531658,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 19.999999999999996,
        "score": 11.827261795600252
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 39.17673354531658
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 18.81407201511874,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 56.44221604535622
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.81407201511874
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.79923075613463,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.06435893507054
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.79923075613463
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 14.038150782601484,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 1.4218116551637516
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.038150782601484
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 71.56044111873955,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 62.298758819612644
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 71.56044111873955
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 62.14187116919363,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 58.246473849360505
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.14187116919363
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 54.783703729457464,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 55.41494097560644
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 54.783703729457464
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 75.33063049195334,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 25.99189147586003
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 75.33063049195334
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 85.2525051117384,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 88.01557985134423
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 85.2525051117384
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 74.88775899012883,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 24.663276970386516
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 74.88775899012883
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8826666666666667,
       "cc": 84.84471574274289,
       "cc_by_type": {
        "choice": 84.84471574274289
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.84471574274289
      },
      "noul": {
       "accuracy": 0.816,
       "cc": 63.19999999999999,
       "cc_by_type": {
        "noul": 63.19999999999999
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 63.19999999999999
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 66.8697791370952,
       "cc_by_type": {
        "score": 66.8697791370952
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 66.8697791370952
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 24.729183321197823,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -50.0,
        "score": 40.18754996359347
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 24.729183321197823
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 91.60263751131986,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 74.80791253395952
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 91.60263751131986
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 33.127488956097665,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 48.16295467317104
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 33.127488956097665
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 15.378424713885506,
       "cc_by_type": {
        "choice": 53.21637426900585,
        "noul": 33.33333333333333,
        "score": -40.41443346068265
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 15.378424713885506
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 36.699498608592535,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 72.41733640548773
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 36.699498608592535
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -31.131294032996152,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 0.0,
        "score": -180.8152657467872
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 56.44044459277766,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 14.28571428571428,
        "score": 55.03561949261869
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.44044459277766
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.56265622049958,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50.0,
        "score": 58.59389200644648
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.56265622049958
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 22.430603549022365,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60.0,
        "score": 68.95847731373377
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 22.430603549022365
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 61.40956000739754,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 56.105910571800045
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 61.40956000739754
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 58.437908295955914,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 41.98039155453445
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 58.437908295955914
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 45.85644368492868,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 34.34352460317314
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 45.85644368492868
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 89.66786126407828,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100.0,
        "score": 69.00358379223486
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 89.66786126407828
      },
      "model_routing": {
       "accuracy": 0.9032258064516129,
       "cc": 79.04214957440841,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100.0,
        "score": 42.97060456738109
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.04214957440841
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 54.225709232648235,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 85.70975725648596
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 54.225709232648235
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 55.27714970469024,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 65.92429962474853
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 55.27714970469024
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 38.09260541997223,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 29.056141383069402
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 38.09260541997223
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 55.326789380349254,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 61.63254205409125
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 55.326789380349254
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.91541807814257,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.74625423442768
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.91541807814257
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.01486620759374,
      91.38185522285914
     ],
     "I": [
      67.16649175722978,
      76.03035129242522
     ],
     "capability": [
      77.48528010616971,
      83.12552326478638
     ],
     "composite": {
      "A": [
       72.52502278899739,
       75.25211552563907
      ],
      "B": [
       71.43810693082546,
       75.36008043389832
      ],
      "C": [
       72.52502278899739,
       75.25211552563907
      ]
     },
     "seed": 74
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.69740052356492,
     "capability": 81.47372650071756,
     "intelligence": 72.25005247787018
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8826666666666667,
      "calibration": 91.16222266166929,
      "cc": 84.84471574274289,
      "cc_by_type": {
       "choice": 84.84471574274289
      },
      "cc_tiered": 84.85498478578386,
      "cc_tiered_open": 84.9873634422428,
      "cc_tiered_sealed": 84.72260612932492,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.84471574274289
     },
     "noul": {
      "accuracy": 0.816,
      "calibration": 90.88108178153634,
      "cc": 63.19999999999999,
      "cc_by_type": {
       "noul": 63.19999999999999
      },
      "cc_tiered": 65.33739484848259,
      "cc_tiered_open": 73.1207729468599,
      "cc_tiered_sealed": 57.554016750105276,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 63.19999999999999
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 90.04889712748917,
      "cc": 66.8697791370952,
      "cc_by_type": {
       "score": 66.8697791370952
      },
      "cc_tiered": 66.5577777993441,
      "cc_tiered_open": 67.54604199888068,
      "cc_tiered_sealed": 65.56951359980751,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 66.8697791370952
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.1699511795255,
    "Emphasis on Accuracy 60:20:20": 70.98745142939559,
    "Emphasis on Cost 20:20:60": 64.58358556228033,
    "Emphasis on Speed 20:60:20": 75.85793898837869,
    "Intelligence only": 72.25005247787018
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tdelab",
   "axes": {
    "calibration": 56.064346266900806,
    "cost": null,
    "intelligence": 5.640752949655241,
    "speed": 93.94749802780368
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.23272817754556263,
      "ece_score": 53.45436449088747,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 53.45436449088747,
      "score_before_time_limit": 53.45436449088747,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.5386666666666666,
      "brier": 0.3053899527599847,
      "ece": 0.22745442238985986,
      "n": 375,
      "score": 54.509115522028026,
      "score_before_time_limit": 54.509115522028026,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.768,
      "interval90_width": 0.5253333333333334,
      "n": 375,
      "nrps": 0.20232630026253115,
      "score": 60.229558787786935,
      "score_before_time_limit": 60.229558787786935,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.29654126199086506
     }
    },
    "score": 56.064346266900806
   },
   "capability": 30.852549608278025,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base answerdotai/ModernBERT-large (the module lists it in UNLISTED_BASE_MODELS; the card declares no base_model); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens: the server reports no token field, so input is counted OFFLINE with the model's own tokenizer on the pod (the counter of the earlier measurement, 672,464 over the full draw, including the 1,024-token truncation) and output is 0 under the 9 Oct 2026 rule (encoder classifier, nothing generated).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "TDE general large v0.1 (tdelab, ModernBERT-large encoder)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (custody #20324, under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's tde-serve, one forward pass.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 7.694898428398389,
    "I_sealed": 3.586607470912093,
    "base": 5.640752949655241,
    "excess": -5.731235812869415,
    "gap": 4.108290957486296,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 23.084695285195167,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.837837837837846,
       "hard": 22.61717521233092,
       "judge": 9.030982201713911,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -42.85714285714286,
       "hard": -47.82608695652174,
       "judge": -40.0,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 15.912047352387827,
       "hard": -26.723417420160132,
       "judge": -8.499775844886592,
       "standard": -0.3084053475133963
      }
     },
     "sealed|choice": {
      "cc": 10.75982241273628,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 6.607369758576881,
       "hard": 6.325893373424297,
       "judge": 13.61410381978454,
       "standard": 17.422484707867554
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -44.827586206896555,
       "hard": -61.81818181818181,
       "judge": -44.61538461538461,
       "standard": -37.31343283582089
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 6.072994815539278,
       "hard": -17.474036712402707,
       "judge": -17.468055468937884,
       "standard": 3.607221118509185
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "tde-general-large",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tdelab/tde-general-large-v0.1@2d697981fc217d50c27d042079549c555bc08e27",
   "not_ranked_because": "unpriced: answerdotai/ModernBERT-large is in the frozen module's UNLISTED_BASE_MODELS (no eligible base-model price); one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.5722543352601156,
    "decisive": 173,
    "decisive_rate": 0.4613333333333333,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 56.064346266900806,
     "cost": null,
     "intelligence": 5.640752949655241,
     "speed": 93.9202043701742
    },
    "board_release": "v1.6.43",
    "capability": 30.852549608278025,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19189397068694233,
     "p50_s_raw": 0.02094698534347117,
     "p95_s_adjusted": 0.21130911619402468,
     "p95_s_raw": 0.03065455809701234
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/tdelab/tde-general-large-v0.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1899578004144132,
    "p50_s_raw": 0.019978900207206607,
    "p95_s_adjusted": 0.2121255909092724,
    "p95_s_raw": 0.031062795454636215
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.3181818181818182,
       "cc": -44.64889836187164,
       "cc_by_type": {
        "choice": 0.0,
        "noul": 0.0,
        "score": -133.94669508561492
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.13861386138613863,
       "cc": -47.882357699574925,
       "cc_by_type": {
        "choice": -14.528101802757151,
        "noul": -92.0,
        "score": -37.11897129596764
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.32673267326732675,
       "cc": -11.452348280944049,
       "cc_by_type": {
        "choice": 5.691056910569103,
        "noul": -40.74074074074075,
        "score": 0.6926389873395
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 27.37802379454176,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -19.999999999999996,
        "score": 2.13407138362528
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 27.37802379454176
      },
      "jaggedness": {
       "accuracy": 0.24444444444444444,
       "cc": -18.086957713428244,
       "cc_by_type": {
        "choice": -5.389797882579387,
        "noul": -56.75675675675676,
        "score": 7.885681499051412
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.41214057507987223,
       "cc": 10.23603012074738,
       "cc_by_type": {
        "choice": 24.716034163951754,
        "noul": -13.4020618556701,
        "score": 19.394118053960486
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 10.23603012074738
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -4.088753649103704,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": 100.0,
        "score": -109.04045449569821
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -35.789280202050165,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100.0,
        "score": -33.141036482439155
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.23272727272727273,
       "cc": -33.64829691573089,
       "cc_by_type": {
        "choice": 4.1408542549722975,
        "noul": -75.0,
        "score": -30.085745002164966
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -19.075778871774677,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -60.0,
        "score": -17.227336615324027
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.4375,
       "cc": 28.095872170439414,
       "cc_by_type": {
        "choice": 28.095872170439414
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 28.095872170439414
      },
      "perturb_pair": {
       "accuracy": 0.390625,
       "cc": 8.380270527016437,
       "cc_by_type": {
        "choice": 21.921921921921925,
        "noul": 0.0,
        "score": 3.2188896591273886
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 8.380270527016437
      },
      "usecase": {
       "accuracy": 0.2636363636363636,
       "cc": -18.700683659499784,
       "cc_by_type": {
        "choice": 15.44117647058827,
        "noul": -58.33333333333333,
        "score": -13.209894115754285
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -31.210698543050498,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": -100.0,
        "score": -55.5368575339134
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.0,
       "cc": -63.15789473684211,
       "cc_by_type": {
        "choice": -26.315789473684216,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.3076923076923077,
       "cc": -10.355750352231984,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -33.333333333333336,
        "score": -14.784608967602253
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.34375,
       "cc": 0.3767855970192109,
       "cc_by_type": {
        "choice": 12.389380530973451,
        "noul": -25.0,
        "score": 13.740976260084182
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0.3767855970192109
      },
      "el": {
       "accuracy": 0.2,
       "cc": -26.806881135846425,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100.0,
        "score": 3.3782392740249634
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.3214596003475239,
       "cc": -10.566948030932052,
       "cc_by_type": {
        "choice": 14.548272537557459,
        "noul": -41.72661870503597,
        "score": -4.522497925317648
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.4838709677419355,
       "cc": -10.39293718864508,
       "cc_by_type": {
        "choice": 41.27243066884176,
        "noul": 0.0,
        "score": -72.451242234777
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -37.651823606267435,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": -47.381700326999024
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.14285714285714285,
       "cc": -47.5054349799874,
       "cc_by_type": {
        "choice": -27.05882352941176,
        "noul": -100.0,
        "score": -15.457481410550432
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.09523809523809523,
       "cc": -79.1298022954612,
       "cc_by_type": {
        "choice": -34.07821229050279,
        "noul": -100.0,
        "score": -103.3111945958808
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.4,
       "cc": -2.400158538865593,
       "cc_by_type": {
        "choice": 15.822291504286824,
        "noul": -33.333333333333336,
        "score": 10.310566212449734
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ja": {
       "accuracy": 0.16,
       "cc": -30.317930523064486,
       "cc_by_type": {
        "choice": -14.376489277204131,
        "noul": -80.0,
        "score": 3.42269770801068
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.1,
       "cc": -60.65000059630072,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -48.61666845556882
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.35714285714285715,
       "cc": -0.3190550228787809,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -33.333333333333336,
        "score": -0.9571650686363453
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -32.239066920314436,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -33.333333333333336,
        "score": -59.48776353150607
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25.0
      },
      "pl": {
       "accuracy": 0.2564102564102564,
       "cc": -31.55678173364039,
       "cc_by_type": {
        "choice": 31.443080829454082,
        "noul": -80.0,
        "score": -46.11342603037525
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.2857142857142857,
       "cc": -19.237636121315095,
       "cc_by_type": {
        "choice": -16.676487919858577,
        "noul": -19.999999999999996,
        "score": -21.036420444086712
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "sv": {
       "accuracy": 0.0,
       "cc": -45.1384804957936,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -100.0,
        "score": -7.755867019295692
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.23076923076923078,
       "cc": -32.401275767040424,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100.0,
        "score": 11.491824872791767
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -52.86352895100111,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100.0,
        "score": -61.81639330461621
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.4,
       "cc": 11.717145125295701,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": 33.33333333333333,
        "score": 3.5130172967910656
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 11.717145125295701
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.3293333333333333,
       "cc": 13.373772938632586,
       "cc_by_type": {
        "choice": 13.373772938632586
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.373772938632586
      },
      "noul": {
       "accuracy": 0.264,
       "cc": -47.199999999999996,
       "cc_by_type": {
        "noul": -47.199999999999996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.31466666666666665,
       "cc": -9.875527857288091,
       "cc_by_type": {
        "score": -9.875527857288091
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -35.61446330550255,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -75.0,
        "score": -67.84338991650765
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -21.70823191758269,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": -33.333333333333336,
        "score": -19.99012018338988
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.2857142857142857,
       "cc": -27.66666238557114,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -100.0,
        "score": -17.959336750209353
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -43.7112086980384,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -49.262281064875424
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.29411764705882354,
       "cc": -34.37213530485037,
       "cc_by_type": {
        "choice": -1.4492753623188428,
        "noul": -100.0,
        "score": -1.6671305522322655
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -109.14827947127996,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100.0,
        "score": -239.39452394843107
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.26666666666666666,
       "cc": -19.614304130752316,
       "cc_by_type": {
        "choice": 31.372549019607845,
        "noul": -71.42857142857143,
        "score": -18.786889983293364
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -20.713998010360054,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -50.0,
        "score": -28.51830065128922
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.06666666666666667,
       "cc": -35.32681576476467,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100.0,
        "score": -1.8137806276273372
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": -4.430408096434276,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -33.333333333333336,
        "score": 9.977869215336721
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.24,
       "cc": -21.949397594541157,
       "cc_by_type": {
        "choice": 18.604651162790695,
        "noul": -55.55555555555556,
        "score": -28.897288390858613
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.1,
       "cc": -30.553913197904084,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -29.29614819586279
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.1111111111111111,
       "cc": -38.5995859651619,
       "cc_by_type": {
        "choice": -15.163147792706338,
        "noul": -60.0,
        "score": -40.63561010277936
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.45161290322580644,
       "cc": 15.210953969333254,
       "cc_by_type": {
        "choice": 32.20779220779223,
        "noul": -6.060606060606055,
        "score": 19.48567576081358
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 15.210953969333254
      },
      "moderation_trust_safety": {
       "accuracy": 0.3333333333333333,
       "cc": -22.93724248098899,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": -11.230153546613808
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.2777777777777778,
       "cc": -28.345864082899254,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -100.0,
        "score": -1.4721326386698985
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.1111111111111111,
       "cc": -57.21913205390808,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100.0,
        "score": -53.430795176502556
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -28.737735566525654,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -66.66666666666667,
        "score": -3.6045110474030473
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.2916666666666667,
       "cc": -24.20298232911786,
       "cc_by_type": {
        "choice": 25.26690391459075,
        "noul": -100.0,
        "score": 2.124149098055661
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      50.262207692557865,
      60.42640133753659
     ],
     "I": [
      3.74912535565067,
      7.628551559966515
     ],
     "capability": [
      27.807284767450415,
      33.53931156979206
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 75
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 56.064346266900806,
     "capability": 30.852549608278025,
     "intelligence": 5.640752949655241
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3293333333333333,
      "calibration": 53.45436449088747,
      "cc": 13.373772938632586,
      "cc_by_type": {
       "choice": 13.373772938632586
      },
      "cc_tiered": 16.922258848965726,
      "cc_tiered_open": 23.084695285195167,
      "cc_tiered_sealed": 10.75982241273628,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.373772938632586
     },
     "noul": {
      "accuracy": 0.264,
      "calibration": 54.509115522028026,
      "cc": -47.199999999999996,
      "cc_by_type": {
       "noul": -47.199999999999996
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.31466666666666665,
      "calibration": 60.229558787786935,
      "cc": -9.875527857288091,
      "cc_by_type": {
       "score": -9.875527857288091
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 53.12005786809952,
     "capability": 29.312187813772837,
     "intelligence": 5.504317759446154
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 83.11679203980711,
    "cost": 64.8083695127977,
    "intelligence": 54.20841827353793,
    "speed": 93.5282853940162
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07938734117954498,
      "ece_score": 84.12253176409101,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 84.12253176409101,
      "score_before_time_limit": 84.12253176409101,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.16388303423251918,
      "ece": 0.11132698482084283,
      "n": 375,
      "score": 77.73460303583144,
      "score_before_time_limit": 77.73460303583144,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.896,
      "interval90_width": 0.3931111111111111,
      "n": 375,
      "nrps": 0.10433692214009276,
      "score": 87.49324131949886,
      "score_before_time_limit": 87.49324131949886,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07289912573496503
     }
    },
    "score": 83.11679203980711
   },
   "capability": 68.66260515667253,
   "composite_ci95": {
    "A": [
     64.28404066727589,
     72.4155360421956
    ],
    "B": [
     59.79574998426001,
     69.04386663255187
    ],
    "C": [
     44.64169490783046,
     68.24627463737896
    ]
   },
   "cost": {
    "basis": "ESTIMATE, v1.5-M2 base-model reference rule: self-hosted open weights, no bookable tariff. Base model Qwen/Qwen3.5-4B (model card base_model). OpenRouter does not list Qwen3.5-4B (host snapshot source/public-docs/OPENROUTER-MODELS.json, fetched 2026-10-12T08:15:26Z, read 2026-10-12). Binding precedent INTERPRETATION-1 I-2/I-3: DeepInfra Qwen/Qwen3.5-4B, USD 0.03/M input, USD 0.15/M output, frozen 25 Sep 2026 price cut-off snapshot (listing later deprecated; frozen reference kept). Logit readout, no generation: server reports 0 output tokens. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.014895 per 1,000 decisions (0.231x the official cost cap USD 0.064595), against USD 0.019658 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01965778
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014895474747474746
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Torchcast Decision 4B (v1.1)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's torchcast_decision server unchanged at tag v1.1.0 with the bundled decision_config.json (bf16, one forward pass reading option letters, calibration temperature 0.9, yes/no commit floor 0.801) on the documented stack torch 2.9.1+cu128 / transformers 5.8.1 / flash-linear-attention 0.5.2 / causal-conv1d 1.7.0 (fast kernels active, CUDA graphs on) Paid fast-lane order 529be5d5 (public). Qwen3.5-4B fine-tune served by the author's own torchcast_decision package. Runtime: official pytorch/pytorch:2.9.1-cuda12.8-cudnn9-devel image (digest-pinned) plus an offline hash-locked overlay of the documented packages (Python 3.11 instead of the documented 3.12). decision_config.json sets a disclosed yes/no commit floor of 0.801: a yes/no probability with max(P, 1-P) below 0.801 is moved to 0.801 on its own side (answer unchanged; same mechanism as h2o-lightning-4b). Prompts over 16,384 tokens keep the head and the tail of the state (documented). Output tokens are 0 (no generation).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 59.789857618232716,
    "I_sealed": 48.626978928843144,
    "base": 54.20841827353793,
    "excess": 1.3233519190338612,
    "gap": 11.162878689389572,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 75.49631276628199,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 62.25228059138094,
       "judge": 76.26895187870798,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 61.89855072463768,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 41.974709363778516,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.04587734174547,
       "hard": 42.40482365626291,
       "judge": 40.031546412698795,
       "standard": 34.993641216445845
      }
     },
     "sealed|choice": {
      "cc": 62.83817359092292,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 79.03430749682339,
       "hard": 55.76500520411702,
       "judge": 62.97747306562194,
       "standard": 68.67749419953597
      }
     },
     "sealed|noul": {
      "cc": 50.59073813474225,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 68.96551724137932,
       "hard": 54.54545454545454,
       "judge": 26.15384615384615,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 32.45202506086426,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 52.45476052442795,
       "hard": 33.44972503305952,
       "judge": 17.565215027813473,
       "standard": 42.78547243426806
      }
     }
    }
   },
   "jevbench_score": 70.67057779139043,
   "key": "torchcast-decision-4b-v1.1",
   "last_measured_on": "2026-10-12",
   "licence": "CC BY-NC 4.0 (weights); base Qwen/Qwen3.5-4B Apache-2.0; serving package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.12",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-4b@f4a34e7aedbf22917f6c85b8c4e03a3da49c1baf",
   "noul_decisive": {
    "acc_among_decisive": 0.784,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.11679203980711,
     "cost": 64.8083695127977,
     "intelligence": 54.20841827353793,
     "speed": 93.62765623388151
    },
    "board_release": "v1.6.43",
    "capability": 68.66260515667253,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 30,
    "capability_rank": 51,
    "composite_ci95": {
     "A": [
      64.29630519793719,
      72.43041609849763
     ],
     "B": [
      59.80423893427847,
      69.05468745077442
     ],
     "C": [
      44.65021194301194,
      68.26018700789584
     ]
    },
    "jevbench_score": 70.68474930403303,
    "measured_in": "v1.6.12",
    "method": "v1.6",
    "open_board_rank": 9,
    "rank": 23,
    "ranks": {
     "A": 23,
     "B": 37,
     "C": 43,
     "capability": 86
    },
    "scores": {
     "A": 70.68474930403303,
     "B": 66.6341381514443,
     "C": 57.69745963223827
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1927332690451294,
     "p50_s_raw": 0.02136663452256471,
     "p95_s_adjusted": 0.2250493116211146,
     "p95_s_raw": 0.037524655810557306
    },
    "views": {
     "Balanced 33:33:33": 67.3279317302704,
     "Emphasis on Accuracy 60:20:20": 61.38534303539167,
     "Emphasis on Cost 20:20:60": 66.2969580344183,
     "Emphasis on Speed 20:60:20": 75.8503894110146,
     "Intelligence only": 54.20841827353793
    }
   },
   "rank": 23,
   "ranked": true,
   "ranks": {
    "A": 23,
    "B": 40,
    "C": 47,
    "capability": 101
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.67057779139043,
    "B": 66.62406257905074,
    "C": 57.68589192793607
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19473725310526788,
    "p50_s_raw": 0.02236862655263394,
    "p95_s_adjusted": 0.22788850581273434,
    "p95_s_raw": 0.03894425290636718
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 60.2110405969951,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 13.9664551243186
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 60.2110405969951
      },
      "calibration_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 49.1657668398499,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 68.0,
        "score": 27.217342937365164
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 49.1657668398499
      },
      "composite_gen": {
       "accuracy": 0.801980198019802,
       "cc": 59.517608397224535,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 25.92592592592593,
        "score": 59.130964306398084
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 59.517608397224535
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 87.56706701867559,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 62.701201056026754
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 87.56706701867559
      },
      "jaggedness": {
       "accuracy": 0.5388888888888889,
       "cc": 35.249975319792384,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": 51.35135135135136,
        "score": 9.25901734142327
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 35.249975319792384
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 69.03721475082187,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 85.56701030927836,
        "score": 50.86575747933239
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.03721475082187
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": 27.405528195442695,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": 100.0,
        "score": -14.557608962059021
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 27.405528195442695
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 16.794675809772134,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": 25.0,
        "score": -0.3891684469722545
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 16.794675809772134
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 48.38591044723288,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": 44.44444444444444,
        "score": 38.86131428558153
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 48.38591044723288
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 50.41309466275076,
       "cc_by_type": {
        "choice": 90.0,
        "noul": 19.999999999999996,
        "score": 41.23928398825228
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 50.41309466275076
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 80.37317882017604,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100.0,
        "score": 62.140557481549116
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.37317882017604
      },
      "usecase": {
       "accuracy": 0.6,
       "cc": 40.995529717776705,
       "cc_by_type": {
        "choice": 57.10784313725491,
        "noul": 33.33333333333333,
        "score": 32.54541268274188
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 40.995529717776705
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -0.033285890015636234,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50.0,
        "score": -54.86176243195167
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 56.06521080351538,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 34.862299077212796
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 56.06521080351538
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 61.46528196168856,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75.0,
        "score": 41.25425296471171
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 61.46528196168856
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 59.346774360020476,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 33.33333333333333,
        "score": 94.98631935566664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 59.346774360020476
      },
      "en": {
       "accuracy": 0.7072111207645526,
       "cc": 53.8082139855539,
       "cc_by_type": {
        "choice": 66.1707814158968,
        "noul": 58.273381294964025,
        "score": 36.98047924580088
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 53.8082139855539
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 61.54934905483835,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 4.2239036082344805
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 61.54934905483835
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 37.48599809267048,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 11.911546190579736
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 37.48599809267048
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 40.01622520990754,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 41.22514621795791
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 40.01622520990754
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 33.23647842158908,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 25.0,
        "score": 24.988764873705794
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 33.23647842158908
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 80.34560570535994,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 55.06643519869871
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 80.34560570535994
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 45.84126362278722,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 39.99999999999999,
        "score": 54.71203550696373
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.84126362278722
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 5.848578443484641,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 17.545735330453926
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 5.848578443484641
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 97.57222944560151,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 92.71668833680451
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 97.57222944560151
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 53.346460809740314,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 78.65410104393959
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 53.346460809740314
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 71.90902061780149,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80.0,
        "score": 43.34449731679848
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 71.90902061780149
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 65.05242711174127,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 48.19204857152315
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.05242711174127
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 40.49448282015345,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 38.07919314131141
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.49448282015345
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 29.93255038880676,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 100.0,
        "score": -44.984957529231906
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.93255038880676
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.16771943867207,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 28.503158316016197
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.16771943867207
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 22.18571449165178,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": -15.928732231259346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 22.18571449165178
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7333333333333333,
       "cc": 65.55617214259745,
       "cc_by_type": {
        "choice": 65.55617214259745
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.55617214259745
      },
      "noul": {
       "accuracy": 0.784,
       "cc": 56.800000000000004,
       "cc_by_type": {
        "noul": 56.800000000000004
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 56.800000000000004
      },
      "score": {
       "accuracy": 0.5653333333333334,
       "cc": 36.55933643117994,
       "cc_by_type": {
        "score": 36.55933643117994
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.55933643117994
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 19.320960055963663,
       "cc_by_type": {
        "choice": 36.0,
        "noul": 25.0,
        "score": -3.0371198321090143
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 19.320960055963663
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 34.22871764612634,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 33.33333333333333,
        "score": 6.619900350387297
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 34.22871764612634
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 11.281778554363948,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": -67.3741765320301
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 11.281778554363948
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 29.702151966686078,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": 37.64446759596467
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 29.702151966686078
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 36.94365711968195,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 35.46865251846613
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 36.94365711968195
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -34.15263657120354,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 33.33333333333333,
        "score": -160.31954493373638
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 56.27374050728449,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 71.42857142857142,
        "score": 17.00049323053695
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.27374050728449
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 22.222092708926684,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": 0.0,
        "score": 29.384048161623255
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 22.222092708926684
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 25.03420356730068,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 19.999999999999996,
        "score": 38.43594403523537
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 25.03420356730068
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 25.182427133062486,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 33.33333333333333,
        "score": 32.14970823716035
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 25.182427133062486
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 53.80625460601258,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 55.55555555555556,
        "score": 29.119022215970546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 53.80625460601258
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 17.361266487762062,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 15.524659678339948
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 17.361266487762062
      },
      "llm_guardrails": {
       "accuracy": 0.7037037037037037,
       "cc": 48.34107386804876,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 36.5395363834169
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 48.34107386804876
      },
      "model_routing": {
       "accuracy": 0.8387096774193549,
       "cc": 69.88393499048401,
       "cc_by_type": {
        "choice": 82.46753246753246,
        "noul": 90.90909090909092,
        "score": 36.27518159482861
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.88393499048401
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 9.85366085849237,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 52.593612134018365
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 9.85366085849237
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 50.4395712119974,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 51.41156414667002
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.4395712119974
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 23.726968533929526,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": -14.040769275058707
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 23.726968533929526
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 19.828044012512837,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": -33.333333333333336,
        "score": 21.802972617248663
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 19.828044012512837
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 90.9303532843901,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 83.4672164368002
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 90.9303532843901
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.22893804404913,
      84.7914607783849
     ],
     "I": [
      48.8894404140232,
      58.33903054464181
     ],
     "capability": [
      63.688569202590465,
      70.97559770691103
     ],
     "composite": {
      "A": [
       64.28404066727589,
       72.4155360421956
      ],
      "B": [
       59.79574998426001,
       69.04386663255187
      ],
      "C": [
       44.64169490783046,
       68.24627463737896
      ]
     },
     "seed": 76
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.11679203980711,
     "capability": 68.66260515667253,
     "intelligence": 54.20841827353793
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 84.12253176409101,
      "cc": 65.55617214259745,
      "cc_by_type": {
       "choice": 65.55617214259745
      },
      "cc_tiered": 69.16724317860246,
      "cc_tiered_open": 75.49631276628199,
      "cc_tiered_sealed": 62.83817359092292,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.55617214259745
     },
     "noul": {
      "accuracy": 0.784,
      "calibration": 77.73460303583144,
      "cc": 56.800000000000004,
      "cc_by_type": {
       "noul": 56.800000000000004
      },
      "cc_tiered": 56.24464442968997,
      "cc_tiered_open": 61.89855072463768,
      "cc_tiered_sealed": 50.59073813474225,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.800000000000004
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 87.49324131949886,
      "cc": 36.55933643117994,
      "cc_by_type": {
       "score": 36.55933643117994
      },
      "cc_tiered": 37.21336721232139,
      "cc_tiered_open": 41.974709363778516,
      "cc_tiered_sealed": 32.45202506086426,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.55933643117994
     }
    },
    "raw_on_A": {
     "calibration": 82.03015024423762,
     "capability": 68.44650938996614,
     "intelligence": 54.862868535694645
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.31078937315633,
    "Emphasis on Accuracy 60:20:20": 61.37679215749364,
    "Emphasis on Cost 20:20:60": 66.28698416463854,
    "Emphasis on Speed 20:60:20": 75.81123736287086,
    "Intelligence only": 54.20841827353793
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 84.9508191193537,
    "cost": 64.8083695127977,
    "intelligence": 60.262524104890716,
    "speed": 90.89431110061412
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08466140495406262,
      "ece_score": 83.06771900918748,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 83.06771900918748,
      "score_before_time_limit": 83.06771900918748,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.12193405070480291,
      "ece": 0.10004788600754745,
      "n": 375,
      "score": 79.99042279849051,
      "score_before_time_limit": 79.99042279849051,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9466666666666667,
      "interval90_width": 0.4382222222222224,
      "n": 375,
      "nrps": 0.08850262773720094,
      "score": 91.7943155503831,
      "score_before_time_limit": 91.7943155503831,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03780553062756855
     }
    },
    "score": 84.9508191193537
   },
   "capability": 72.6066716121222,
   "composite_ci95": {
    "A": [
     70.2623629737475,
     74.32797026605473
    ],
    "B": [
     66.50569125845816,
     72.00839961142943
    ],
    "C": [
     58.76312636697478,
     74.32797026605473
    ]
   },
   "cost": {
    "basis": "Self-hosted open weights, no bookable tariff. v1.5-M2 base-model reference rule: base Qwen/Qwen3.5-4B (model card base_model, finetune). OpenRouter snapshot of 2026-10-18 lists no Qwen3.5-4B; binding precedent INTERPRETATION-1 I-2/I-3 uses DeepInfra Qwen/Qwen3.5-4B at USD 0.03/M input and USD 0.15/M output from the frozen 25 Sep 2026 price cut-off snapshot (read 2026-10-18). No-generation readout: output tokens expected 0. Frozen reference deepinfra:Qwen/Qwen3.5-4B; 982,889 input / 0 output tokens over all 1,500 decisions (USD 0.0196578 per 1,000 over all 1,500 items), inside the official cap. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.014895 per 1,000 decisions (0.231x the official cost cap USD 0.064595), against USD 0.019658 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.019657780000000003
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01489547474747475
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Torchcast Decision 4B (v1.2)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-18",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Verda FIN-01 A100 SXM4 80GB GPU VM, offline read-only container (--network none); author's torchcast_decision server unchanged at tag v1.2.0 with the bundled decision_config.json (bf16, one forward pass reading option letters, non-causal full attention as trained, calibration temperature 1.0, yes/no commit floor 0.801) on the documented stack torch 2.9.1+cu128 / transformers 5.8.1 / flash-linear-attention 0.5.2 / causal-conv1d 1.7.0 (fast kernels active; no CUDA graphs in non-causal mode)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 62.74872292249266,
    "I_sealed": 57.77632528728878,
    "base": 60.262524104890716,
    "excess": -4.867129135151828,
    "gap": 4.972397635203883,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.16814944751415,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 73.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 42.47898550353872,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.38410762065289,
       "hard": 52.07017376518772,
       "judge": 43.985586059358674,
       "standard": 19.084147087953696
      }
     },
     "sealed|choice": {
      "cc": 70.81428900718828,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 69.81612119810339,
       "judge": 63.85896180215475,
       "standard": 75.32166209660409
      }
     },
     "sealed|noul": {
      "cc": 63.36387030365415,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 60.00000000000001,
       "judge": 60.00000000000001,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 39.15081655102391,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 68.46129282933187,
       "hard": 34.48432889360359,
       "judge": 35.38697723076133,
       "standard": 39.474312707104446
      }
     }
    }
   },
   "jevbench_score": 72.99606771025861,
   "key": "torchcast-decision-4b-v1.2",
   "last_measured_on": "2026-10-18",
   "licence": "Apache-2.0 (weights, from v1.2); base Qwen/Qwen3.5-4B Apache-2.0; serving package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.29",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-4b@d54b248f2643df847979ca15638ee502ec79b788",
   "noul_decisive": {
    "acc_among_decisive": 0.8373333333333334,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.9508191193537,
     "cost": 64.8083695127977,
     "intelligence": 60.262524104890716,
     "speed": 91.12622136196717
    },
    "board_release": "v1.6.43",
    "capability": 72.6066716121222,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 16,
    "capability_rank": 33,
    "composite_ci95": {
     "A": [
      70.29693613333056,
      74.3666613117086
     ],
     "B": [
      66.53046829296893,
      72.03744728597768
     ],
     "C": [
      58.79194113894132,
      74.3666613117086
     ]
    },
    "jevbench_score": 73.03338420137015,
    "measured_in": "v1.6.29",
    "method": "v1.6",
    "open_board_rank": 4,
    "rank": 14,
    "ranks": {
     "A": 14,
     "B": 20,
     "C": 14,
     "capability": 59
    },
    "scores": {
     "A": 73.03338420137015,
     "B": 70.06379430073542,
     "C": 73.03338420137015
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2749862310000367,
     "p50_s_raw": 0.06249311550001835,
     "p95_s_adjusted": 0.28058659270005815,
     "p95_s_raw": 0.06529329635002909
    },
    "views": {
     "Balanced 33:33:33": 69.77075738979846,
     "Emphasis on Accuracy 60:20:20": 65.62878799360888,
     "Emphasis on Cost 20:20:60": 67.69731928870637,
     "Emphasis on Speed 20:60:20": 76.98758510619885,
     "Intelligence only": 60.262524104890716
    }
   },
   "rank": 15,
   "ranked": true,
   "ranks": {
    "A": 15,
    "B": 22,
    "C": 15,
    "capability": 71
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.99606771025861,
    "B": 70.03631619161014,
    "C": 72.99606771025861
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2755932589999134,
    "p50_s_raw": 0.06279662949995668,
    "p95_s_adjusted": 0.2953250589999356,
    "p95_s_raw": 0.0726625294999678
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 78.58208761447007,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 44.07959617674351
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 78.58208761447007
      },
      "calibration_gen": {
       "accuracy": 0.504950495049505,
       "cc": 29.652491168192714,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": 52.0,
        "score": 3.765532889519818
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 29.652491168192714
      },
      "composite_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 69.68323903439195,
       "cc_by_type": {
        "choice": 86.9918699186992,
        "noul": 62.962962962962955,
        "score": 59.09488422151368
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 69.68323903439195
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 92.44612065592298,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 77.33836196776895
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 92.44612065592298
      },
      "jaggedness": {
       "accuracy": 0.6055555555555555,
       "cc": 40.56092887250238,
       "cc_by_type": {
        "choice": 55.24542829643889,
        "noul": 45.945945945945944,
        "score": 20.49141237512231
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 40.56092887250238
      },
      "llm_router": {
       "accuracy": 0.8626198083067093,
       "cc": 75.58422268224218,
       "cc_by_type": {
        "choice": 83.35828123624195,
        "noul": 93.81443298969072,
        "score": 49.5799538207939
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 75.58422268224218
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -5.097572874998855,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0.0,
        "score": -24.970137979835272
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 4.073147067528581,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": 0.0,
        "score": -1.1826206530843653
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 4.073147067528581
      },
      "multilingual": {
       "accuracy": 0.7672727272727272,
       "cc": 64.00830885734634,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 72.22222222222223,
        "score": 48.169186253957605
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 64.00830885734634
      },
      "negation_pair_gen": {
       "accuracy": 0.7790697674418605,
       "cc": 47.57400302837292,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 22.722009085118767
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 47.57400302837292
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 80.37903990692341,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100.0,
        "score": 65.16114374479423
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.37903990692341
      },
      "usecase": {
       "accuracy": 0.7,
       "cc": 57.6090573165931,
       "cc_by_type": {
        "choice": 68.1372549019608,
        "noul": 58.33333333333333,
        "score": 46.35658371448517
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 57.6090573165931
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 9.22238439439781,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50.0,
        "score": -46.142370626330376
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 9.22238439439781
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 45.79680121584361,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 24.7943975031376
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 45.79680121584361
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 77.70007818419383,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100.0,
        "score": 72.92324340213901
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.70007818419383
      },
      "el": {
       "accuracy": 0.8,
       "cc": 74.26271122284257,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 89.64102007448673
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 74.26271122284257
      },
      "en": {
       "accuracy": 0.7471763683753258,
       "cc": 60.10371398764253,
       "cc_by_type": {
        "choice": 72.98055918282665,
        "noul": 66.18705035971225,
        "score": 41.143532420388674
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 60.10371398764253
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 64.64149013424526,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 3.712398624595481
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 64.64149013424526
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 58.4795453997909,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 74.89218811194102
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 58.4795453997909
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 77.57866686911653,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 53.91247119558486
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 77.57866686911653
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 38.57772998083417,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 25.0,
        "score": 41.01251955144107
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 38.57772998083417
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 71.19687350509501,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 27.620238597903914
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.19687350509501
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 62.53605931537678,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 100.0,
        "score": 16.20230026543141
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 62.53605931537678
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 61.26924456890534,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0,
        "score": 28.252178151160468
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 61.26924456890534
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 96.12707554673155,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 88.38122664019465
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 96.12707554673155
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 46.541321476730594,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 58.238683044910424
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.541321476730594
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 76.23616141039446,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 63.94335515797137
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 76.23616141039446
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 68.46010565834278,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 37.20117731680793
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 68.46010565834278
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 65.46355665099959,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 61.92258484661579
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 65.46355665099959
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 37.91534536977069,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50.0,
        "score": -14.514833455905318
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 37.91534536977069
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 67.99341252934869,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 36.23830210417509
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.99341252934869
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 31.75490779026978,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 12.77884766459465
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.75490779026978
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7853333333333333,
       "cc": 72.27271857479094,
       "cc_by_type": {
        "choice": 72.27271857479094
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 72.27271857479094
      },
      "noul": {
       "accuracy": 0.8373333333333334,
       "cc": 67.46666666666667,
       "cc_by_type": {
        "noul": 67.46666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 67.46666666666667
      },
      "score": {
       "accuracy": 0.584,
       "cc": 41.326578005993696,
       "cc_by_type": {
        "score": 41.326578005993696
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 41.326578005993696
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 54.51220458804392,
       "cc_by_type": {
        "choice": 52.0,
        "noul": 75.0,
        "score": 36.536613764131765
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 54.51220458804392
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 19.414032579676796,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 11.111111111111116,
        "score": 3.031607745931697
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 19.414032579676796
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 47.98302937571595,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 42.72957593202591
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 47.98302937571595
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 50.4289331415679,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 100.0,
        "score": 21.462238021194935
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 50.4289331415679
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 59.47084257633375,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 59.57194801885635
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.47084257633375
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -56.08570821762897,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 66.66666666666667,
        "score": -259.45209320634604
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 50.752697382203884,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 29.008792426723684
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 50.752697382203884
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 42.737556232348936,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": 50.0,
        "score": 30.477477059416145
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 42.737556232348936
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 34.180989510345235,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 45.04296853103571
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.180989510345235
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 53.74050987098581,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 33.09876016256486
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 53.74050987098581
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 61.064515449092106,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 39.26589776846497
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 61.064515449092106
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 11.797581376060576,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -1.1663956567645073
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.797581376060576
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 50.15312163699627,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 53.49199446953005
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 50.15312163699627
      },
      "model_routing": {
       "accuracy": 0.8663594470046083,
       "cc": 74.8931310594956,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 96.96969696969697,
        "score": 37.06034555943917
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.8931310594956
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 49.724431860882,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 72.20592514118727
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 49.724431860882
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 82.1798596706628,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": 63.25267093399397
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 82.1798596706628
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 39.6143521594331,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50.0,
        "score": 13.178031847757444
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 39.6143521594331
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 47.91103234199653,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 33.33333333333333,
        "score": 24.892517315844675
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 47.91103234199653
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 92.8233635346338,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 89.1462471875313
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.8233635346338
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.96169960479254,
      86.52145783591912
     ],
     "I": [
      54.797400191098326,
      64.34833891333052
     ],
     "capability": [
      68.1468487749732,
      74.61557349985053
     ],
     "composite": {
      "A": [
       70.2623629737475,
       74.32797026605473
      ],
      "B": [
       66.50569125845816,
       72.00839961142943
      ],
      "C": [
       58.76312636697478,
       74.32797026605473
      ]
     },
     "seed": 77
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.9508191193537,
     "capability": 72.6066716121222,
     "intelligence": 60.262524104890716
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7853333333333333,
      "calibration": 83.06771900918748,
      "cc": 72.27271857479094,
      "cc_by_type": {
       "choice": 72.27271857479094
      },
      "cc_tiered": 71.49121922735122,
      "cc_tiered_open": 72.16814944751415,
      "cc_tiered_sealed": 70.81428900718828,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 72.27271857479094
     },
     "noul": {
      "accuracy": 0.8373333333333334,
      "calibration": 79.99042279849051,
      "cc": 67.46666666666667,
      "cc_by_type": {
       "noul": 67.46666666666667
      },
      "cc_tiered": 68.48145206003963,
      "cc_tiered_open": 73.59903381642512,
      "cc_tiered_sealed": 63.36387030365415,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 67.46666666666667
     },
     "score": {
      "accuracy": 0.584,
      "calibration": 91.7943155503831,
      "cc": 41.326578005993696,
      "cc_by_type": {
       "score": 41.326578005993696
      },
      "cc_tiered": 40.814901027281316,
      "cc_tiered_open": 42.47898550353872,
      "cc_tiered_sealed": 39.15081655102391,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.326578005993696
     }
    },
    "raw_on_A": {
     "calibration": 83.43579348904761,
     "capability": 72.66452279180986,
     "intelligence": 61.893252094572105
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.72535457117256,
    "Emphasis on Accuracy 60:20:20": 65.60467788938145,
    "Emphasis on Cost 20:20:60": 67.67166569368591,
    "Emphasis on Speed 20:60:20": 76.88814279279627,
    "Intelligence only": 60.262524104890716
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 87.84716845590987,
    "cost": 64.8083695127977,
    "intelligence": 67.14748494108278,
    "speed": 92.44891117449383
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06325270940387061,
      "ece_score": 87.34945811922587,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.34945811922587,
      "score_before_time_limit": 87.34945811922587,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.1069128488645628,
      "ece": 0.06240741177362208,
      "n": 375,
      "score": 87.51851764527558,
      "score_before_time_limit": 87.51851764527558,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.928,
      "interval90_width": 0.3104444444444443,
      "n": 375,
      "nrps": 0.0771279973557552,
      "score": 88.67352960322816,
      "score_before_time_limit": 88.67352960322816,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.07470070528984067
     }
    },
    "score": 87.84716845590987
   },
   "capability": 77.49732669849632,
   "composite_ci95": {
    "A": [
     73.57382388897746,
     77.25534470961512
    ],
    "B": [
     70.76344984805432,
     75.82283743435625
    ],
    "C": [
     73.56894204574836,
     77.25534470961512
    ]
   },
   "cost": {
    "basis": "Self-hosted open weights, no bookable tariff. v1.5-M2 base-model reference rule: base Qwen/Qwen3.5-4B (model card base_model, finetune). OpenRouter snapshot of 2026-10-23 lists no Qwen3.5-4B; binding precedent INTERPRETATION-1 I-2/I-3 uses DeepInfra Qwen/Qwen3.5-4B at USD 0.03/M input and USD 0.15/M output from the frozen 25 Sep 2026 price cut-off snapshot (read 2026-10-23), through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7), the same reference as the live v1.2 row. ESTIMATE, not charged, not a quote from the author or any host. No-generation readout: output tokens 0. Frozen reference deepinfra:Qwen/Qwen3.5-4B; 982,889 input / 0 output tokens over all 1,500 decisions (USD 0.0196578 per 1,000 over all 1,500 items; the 15 truncated prompts count at their truncated size of about 16,370 tokens each), inside the official cap. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items that count for no row; 737,326 input / 0 output tokens on those 1,485 items. This row: USD 0.014895 per 1,000 decisions (0.231x the official cost cap USD 0.064595), against USD 0.019658 over all 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.019657780000000003
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01489547474747475
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Torchcast Decision 4B (v1.3)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM (1A100.22V, FIN-02, Finland, EU, pay-as-you-go), 1x NVIDIA A100 SXM4 80GB; offline read-only container (network none); author's torchcast_decision server unchanged at tag v1.3.0 with the bundled decision_config.json (bf16, one forward pass reading option letters, non-causal full attention as trained, calibration temperature 0.9, yes/no commit floor 0.801) on the documented stack torch 2.9.1+cu128 / transformers 5.8.1 / flash-linear-attention 0.5.2 / causal-conv1d 1.7.0 (fast kernels active; CUDA graphs with a key-padding mask in non-causal mode). Driven by the unchanged self-hosted runner on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...), measured 23 Oct 2026; raw latency p50 0.034 s, p95 0.056 s, maximum 7.73 s. The standard self-hosted x2 + 0.15 s adjustment applies (assumption, not measured).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 69.73993545583312,
    "I_sealed": 64.55503442633244,
    "base": 67.14748494108278,
    "excess": -4.654625740855032,
    "gap": 5.184901029500679,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 81.67361775115984,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 84.17930125247199,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 80.09316770186336,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 91.30434782608697,
       "judge": 50.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 47.45302091447617,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.35515078027366,
       "hard": 59.61562740569072,
       "judge": 50.73096446053644,
       "standard": 23.75982768005792
      }
     },
     "sealed|choice": {
      "cc": 72.9759407483175,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 68.77529779114143,
       "judge": 67.384916748286,
       "standard": 81.01666315123391
      }
     },
     "sealed|noul": {
      "cc": 70.89886952985255,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 65.45454545454545,
       "judge": 78.46153846153847,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 49.79029300082726,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.76877798198868,
       "hard": 45.662863577821,
       "judge": 48.12725276024139,
       "standard": 50.05046971713786
      }
     }
    }
   },
   "jevbench_score": 76.15727509375536,
   "key": "torchcast-decision-4b-v1.3",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (weights, from v1.2); base Qwen/Qwen3.5-4B Apache-2.0; serving package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.42",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-4b@004acea2e23056bc79415cfa69b9f3ea4254e6d0",
   "noul_decisive": {
    "acc_among_decisive": 0.8613333333333333,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.84716845590987,
     "cost": 64.8083695127977,
     "intelligence": 67.14748494108278,
     "speed": 92.63835078333227
    },
    "board_release": "v1.6.43",
    "capability": 77.49732669849632,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 11,
    "capability_rank": 20,
    "composite_ci95": {
     "A": [
      73.60377007858608,
      77.28836347482343
     ],
     "B": [
      70.78560945953136,
      75.84827959180653
     ],
     "C": [
      73.59888426077293,
      77.28836347482343
     ]
    },
    "jevbench_score": 76.18936170949911,
    "measured_in": "v1.6.42",
    "method": "v1.6",
    "open_board_rank": 1,
    "rank": 3,
    "ranks": {
     "A": 3,
     "B": 7,
     "C": 3,
     "capability": 36
    },
    "scores": {
     "A": 76.18936170949911,
     "B": 74.19128695067,
     "C": 76.18936170949911
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21561166599999523,
     "p50_s_raw": 0.03280583299999762,
     "p95_s_adjusted": 0.25263450489999856,
     "p95_s_raw": 0.051317252449999275
    },
    "views": {
     "Balanced 33:33:33": 72.96187960528026,
     "Emphasis on Accuracy 60:20:20": 70.51933253922925,
     "Emphasis on Cost 20:20:60": 69.46608174549742,
     "Emphasis on Speed 20:60:20": 79.73630555208705,
     "Intelligence only": 67.14748494108278
    }
   },
   "rank": 4,
   "ranked": true,
   "ranks": {
    "A": 4,
    "B": 9,
    "C": 4,
    "capability": 43
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 76.15727509375536,
    "B": 74.1669440986912,
    "C": 76.15727509375536
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21713105699998322,
    "p50_s_raw": 0.033565528499991615,
    "p95_s_adjusted": 0.26205167260007445,
    "p95_s_raw": 0.05602583630003721
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "004acea2e23056bc79415cfa69b9f3ea4254e6d0 Paid fast-lane order 9494e9ea (public). Qwen3.5-4B fine-tune served by the author's own torchcast_decision package, measured as shipped: the author's server unchanged at tag v1.3.0 with its bundled decision_config.json (repository torchcast-ai/torchcast-decision-4b at the pinned commit above). Same runtime as the v1.2 row: official pytorch/pytorch:2.9.1-cuda12.8-cudnn9-devel image (digest-pinned) plus an offline hash-locked overlay of the documented packages (transformers 5.8.1, flash-linear-attention 0.5.2, causal-conv1d 1.7.0), on Python 3.11 instead of the documented 3.12. decision_config.json sets attention_mode noncausal_full_attention (the full-attention layers see the whole prompt in both directions, as trained; causal runtimes such as vLLM give different probabilities) and a disclosed yes/no commit floor of 0.801: a yes/no probability with max(P, 1-P) below 0.801 is moved to 0.801 on its own side (answer unchanged; same mechanism as h2o-lightning-4b, v1.1 and v1.2). New in v1.3: new fine-tuned weights; calibration temperature 0.9 (v1.2 used 1.0); CUDA graphs in the non-causal mode (speed only: graph and eager readouts were bit-identical in our check). Prompts over 16,384 tokens keep the head and the tail of the state (documented). Output tokens are 0 (no generation). All 1,500 items answered, none refused; the 15 states longer than 16,384 tokens are cut by the model to head and tail and scored as answered, as for earlier truncated rows. No decision took more than 10 seconds (maximum 7.73 s). The cost is a labelled estimate on the Qwen3.5-4B base-model reference, not a price the author or a host charges. Self-hosted speed adjustment as for every self-hosted row; the device class (A100 SXM4 80GB) is the same as for v1.2. v1.3 is a new row next to v1.2; v1.2 remains listed as its own version.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1.0,
       "cc": 87.74496676245083,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 63.2349002873525
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.74496676245083
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 25.445755345604557,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": 60.00000000000001,
        "score": -4.129329933493864
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 25.445755345604557
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 82.96222558951331,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 62.79814921659776
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 82.96222558951331
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.57910175574943,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 90.7373052672483
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.57910175574943
      },
      "jaggedness": {
       "accuracy": 0.7166666666666667,
       "cc": 56.95838990018219,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": 56.75675675675676,
        "score": 50.210809478919735
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 56.95838990018219
      },
      "llm_router": {
       "accuracy": 0.8753993610223643,
       "cc": 76.73558744630976,
       "cc_by_type": {
        "choice": 88.1130580258871,
        "noul": 87.62886597938144,
        "score": 54.46483833366074
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 76.73558744630976
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": -3.1899473094131046,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": 0.0,
        "score": -32.15048708952963
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 34.50350246279871,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": 25.0,
        "score": 40.36617749148892
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 34.50350246279871
      },
      "multilingual": {
       "accuracy": 0.8,
       "cc": 68.80518324507379,
       "cc_by_type": {
        "choice": 75.54613628953375,
        "noul": 72.22222222222223,
        "score": 58.64719122346542
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.80518324507379
      },
      "negation_pair_gen": {
       "accuracy": 0.8372093023255814,
       "cc": 61.92028089814509,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 66.66666666666667,
        "score": 25.760842694435272
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 61.92028089814509
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 83.54813299356996,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100.0,
        "score": 71.66542000173091
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.54813299356996
      },
      "usecase": {
       "accuracy": 0.7181818181818181,
       "cc": 59.913445939313554,
       "cc_by_type": {
        "choice": 71.81372549019609,
        "noul": 58.33333333333333,
        "score": 49.59327899441123
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.913445939313554
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 27.179843956497706,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50.0,
        "score": 7.730008059969307
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.179843956497706
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 76.52840979397297,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 29.5852293819189
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 76.52840979397297
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 67.8305005650512,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 50.0,
        "score": 85.34990877479962
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 67.8305005650512
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 65.5464196197277,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 96.82547859847546
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 65.5464196197277
      },
      "en": {
       "accuracy": 0.7836663770634231,
       "cc": 65.85640801672774,
       "cc_by_type": {
        "choice": 75.83627243992628,
        "noul": 72.66187050359711,
        "score": 49.071081106659854
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 65.85640801672774
      },
      "es": {
       "accuracy": 0.9032258064516129,
       "cc": 71.49641471093192,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 14.489244132795754
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.49641471093192
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 52.50995210855385,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 89.77029348413149
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 52.50995210855385
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 87.62061012451325,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 62.86183037353974
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 87.62061012451325
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 69.5539278240772,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75.0,
        "score": 50.421560008544475
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 69.5539278240772
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 1.0,
       "cc": 82.31194702287515,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 46.93584106862545
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 82.31194702287515
      },
      "ja": {
       "accuracy": 0.76,
       "cc": 61.54082807954149,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 100.0,
        "score": 27.51366771757603
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 61.54082807954149
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 78.84328923232499,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 36.52986769697496
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 78.84328923232499
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 98.48797537075977,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.4639261122793
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.48797537075977
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 55.05285323515513,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 57.79925234615803
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 55.05285323515513
      },
      "no": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 100.0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 70.86230835382561,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 60.00000000000001,
        "score": 67.82179598826481
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 70.86230835382561
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 75.384975232108,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 57.97578603810361
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.384975232108
      },
      "sv": {
       "accuracy": 0.8181818181818182,
       "cc": 70.33653784602723,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 76.54152843169868
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 70.33653784602723
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 36.94476888960471,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50.0,
        "score": 26.051697973161936
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 36.94476888960471
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 64.78031434979971,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 26.599007565528154
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 64.78031434979971
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 32.665765560240864,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 15.511420974507917
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 32.665765560240864
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.812,
       "cc": 75.7171013605312,
       "cc_by_type": {
        "choice": 75.7171013605312
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.7171013605312
      },
      "noul": {
       "accuracy": 0.8613333333333333,
       "cc": 72.26666666666665,
       "cc_by_type": {
        "noul": 72.26666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 72.26666666666665
      },
      "score": {
       "accuracy": 0.6586666666666666,
       "cc": 50.36312555406744,
       "cc_by_type": {
        "score": 50.36312555406744
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.36312555406744
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 52.14026099443436,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 50.0,
        "score": 38.420782983303056
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 52.14026099443436
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 24.324497869768763,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 33.33333333333333,
        "score": 14.174321766656195
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 24.324497869768763
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 44.95963108067105,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 33.65938104689121
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.95963108067105
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 23.877798360690733,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": 8.475500345230103
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 23.877798360690733
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 40.31542722571074,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 68.77236863365397
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.31542722571074
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 40.31629620404986,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 100.0,
        "score": -53.89387868344788
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 40.31629620404986
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 57.58634953553787,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 42.85714285714286,
        "score": 29.901905749470746
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.58634953553787
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 49.580198070423315,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 50.0,
        "score": 40.55244090116542
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 49.580198070423315
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 62.75826825070436,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 69.94147141877976
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 62.75826825070436
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 52.16925909140334,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 28.385007823817443
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 52.16925909140334
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.95887685307294,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 37.72675975818526
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.95887685307294
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 10.405704752945985,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 26.916038990020752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 10.405704752945985
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 59.1188646790882,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 57.356594037264564
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.1188646790882
      },
      "model_routing": {
       "accuracy": 0.8709677419354839,
       "cc": 74.91323371602249,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 87.87878787878789,
        "score": 42.70506911343546
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.91323371602249
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 55.12660241169061,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 88.41243679361308
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 55.12660241169061
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 82.28589319945957,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": 63.57077152038424
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 82.28589319945957
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 61.39301033055477,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 50.0,
        "score": 48.95735611481702
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 61.39301033055477
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 58.56002850862949,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 66.66666666666667,
        "score": 37.9989261055986
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 58.56002850862949
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 97.56474335342972,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 92.69423006028917
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.56474335342972
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.80049527648649,
      88.86935442791973
     ],
     "I": [
      61.118891454923116,
      70.91801982503955
     ],
     "capability": [
      72.88106875800261,
      79.21481156880128
     ],
     "composite": {
      "A": [
       73.57382388897746,
       77.25534470961512
      ],
      "B": [
       70.76344984805432,
       75.82283743435625
      ],
      "C": [
       73.56894204574836,
       77.25534470961512
      ]
     },
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.84716845590987,
     "capability": 77.49732669849632,
     "intelligence": 67.14748494108278
    },
    "per_type": {
     "choice": {
      "accuracy": 0.812,
      "calibration": 87.34945811922587,
      "cc": 75.7171013605312,
      "cc_by_type": {
       "choice": 75.7171013605312
      },
      "cc_tiered": 77.32477924973867,
      "cc_tiered_open": 81.67361775115984,
      "cc_tiered_sealed": 72.9759407483175,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.7171013605312
     },
     "noul": {
      "accuracy": 0.8613333333333333,
      "calibration": 87.51851764527558,
      "cc": 72.26666666666665,
      "cc_by_type": {
       "noul": 72.26666666666665
      },
      "cc_tiered": 75.49601861585796,
      "cc_tiered_open": 80.09316770186336,
      "cc_tiered_sealed": 70.89886952985255,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 72.26666666666665
     },
     "score": {
      "accuracy": 0.6586666666666666,
      "calibration": 88.67352960322816,
      "cc": 50.36312555406744,
      "cc_by_type": {
       "score": 50.36312555406744
      },
      "cc_tiered": 48.621656957651716,
      "cc_tiered_open": 47.45302091447617,
      "cc_tiered_sealed": 49.79029300082726,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.36312555406744
     }
    },
    "raw_on_A": {
     "calibration": 82.20877716936431,
     "capability": 74.21943016819003,
     "intelligence": 66.23008316701576
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.92264987037383,
    "Emphasis on Accuracy 60:20:20": 70.4973393055883,
    "Emphasis on Cost 20:20:60": 69.44474047203342,
    "Emphasis on Speed 20:60:20": 79.6520143405633,
    "Intelligence only": 67.14748494108278
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "MachineFi",
   "axes": {
    "calibration": 85.64268754215436,
    "cost": null,
    "intelligence": 28.409112519894506,
    "speed": 76.18058448046628
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07299797297297295,
      "ece_score": 85.40040540540541,
      "mean_tvd": null,
      "n": 674,
      "n_ece_hard": 296,
      "n_tvd": 0,
      "score": 85.40040540540541,
      "score_before_time_limit": 85.40040540540541,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3907103825136612,
      "brier": 0.13999579877049192,
      "ece": 0.09047732240437158,
      "n": 366,
      "score": 81.90453551912569,
      "score_before_time_limit": 81.90453551912569,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9413407821229051,
      "interval90_width": 0.513268156424581,
      "n": 358,
      "nrps": 0.10858002406191804,
      "score": 89.62312170193202,
      "score_before_time_limit": 89.62312170193202,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04947877094972075
     }
    },
    "score": 85.64268754215436
   },
   "capability": 57.025900031024435,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED under the frozen pricing_v15 path (sha256 dcfcecb06525ce4a5ece37400ea261b97d2fe5d41155f3cd02b03cac86572835, the module in force since v1.6.21; also under its predecessor 32117d754104b3ead8bebdc9da5f8e0116a7550f4316cfc6874fbe6234c2d72b, against which it was first checked): MachineFi publishes a usage tariff for Trio-Spark of USD 0.042 per 1M billed input tokens with no output charge (docs/visual.md and the README of machinefi/trio-spark), but `trio-spark-v1.3` is in neither AGE_QUALIFIED_API nor LAUNCH_LIST_PRICE_API, and the system declares no base model and releases no weights, so there is no base-model market reference to fall back on. The frozen path therefore returns no eligible price (the same outcome as sage-1.4.0, and as subq-snap-preview before CoS deputy #19174).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Trio-Spark v1.3 (MachineFi)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted `platform.machinefi.com/v1/systemone` API, model `trio-spark-v1.3`; latency measured from Sandy (Germany) without API adjustment — the submitter explicitly asked for it to be timed as a hosted endpoint with the network included (over all 1,398 answered requests p50 1.346 s, p95 1.855 s wall; the Speed axis uses the official per-item latency subset; loadavg samples for the whole run are in the measurement receipts on file). Full S1200+P300 draw of `v1.6-regular-20261012` through the unchanged `run_api_full_v161.py` (sha256 `eceea17e…`) and the unchanged `jevbench.adapters.typesafe` (sha256 `780f8794…`); the full-set exposure gate logged all 1,200 sealed items to provider `machinefi` before the first request (custody #19732; this draw's sealed items were already past the two-provider retirement threshold, so the admission cost the reserve nothing). Native System One answer fields on all three question types, nothing generated — with one honesty note the independent review asked for (P2-3): MachineFi's public docs describe only `choice`, so `noul` and `score` are documented here by observation, not by their documentation. The adapter's `noul` → P(yes) orientation is not assumed either: on the 231 PUBLIC items this mapping scores 80.6 % (54 of 67 noul items) against 19.4 % if it were inverted, which settles the direction without touching a sealed item (`output_tokens` 0 on every answered row, 573,254 input tokens over the full set). Hosting implementation and deployed weights were not independently inspected; no weights or training code are published. Regular-queue request `bh-submit:216579c8` (ref BBAEFC3A), measured with the submitter's own API key. 102 of 1,500 requests returned HTTP 422 `request_refused`, for three reasons given in MachineFi's own error messages: 58 because the question has 9 options and Trio-Spark supports at most 8 (\"Question decision has 9 options; Trio-Spark supports at most 8.\" - every 9-option item of this draw), 43 with \"Input exceeds the supported token limit.\" (MachineFi's documented cap of 1,024 model input tokens; answered prompts reach at most 955 billed input tokens), and 1 with \"The decision input cannot be evaluated.\" By type: 76 choice, 17 score, 9 noul. The runner never re-sends a failed item and no request was truncated or shortened, so all 102 are scored as wrong in the full 1,500 denominator. No further requests failed for other reasons. The refusal count was pre-declared at 27–193 before the run, expecting token-limit refusals only; the 8-option limit was not foreseen there, and the add-requests hand-over attributed all 102 to the token cap, which the raw's own error messages correct. The pre-data text is §4 of the pre-registration at sha256 `e4340cdb…`, published on the agent board at 22:45Z and 22:55Z — before the dispatch began at 23:03:30Z — and preserved byte-exactly in the measurement receipts on file; the file itself was amended twice afterwards and now hashes `2fbc0ceb…`.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 34.5165059024833,
    "I_sealed": 22.30171913730571,
    "base": 28.409112519894506,
    "excess": 2.3752599948218815,
    "gap": 12.214786765177593,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 48.51140713192591,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 45.513513513513516,
       "hard": 45.265806857502355,
       "judge": 44.62755438365195,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 23.29882677708765,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 21.739130434782616,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 31.739283798436347,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.26121212121212,
       "hard": 24.1500925925926,
       "judge": 29.718813559322033,
       "standard": 39.187407407407406
      }
     },
     "sealed|choice": {
      "cc": 50.113978310534534,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 52.350698856416784,
       "hard": 48.99965305886434,
       "judge": 52.39960822722822,
       "standard": 47.79582366589328
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 27.586206896551737,
       "hard": -12.727272727272732,
       "judge": -16.92307692307692,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 16.791179101382603,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 39.64219298245615,
       "hard": -3.60504225352114,
       "judge": 27.741770833333312,
       "standard": 29.732227272727265
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "trio-spark-v1.3",
   "last_measured_on": "2026-10-15",
   "licence": "Proprietary hosted API (no weights or training code released; the repo's own client code is Apache-2.0)",
   "listing": "listed",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:platform.machinefi.com/v1/systemone#trio-spark-v1.3",
   "not_ranked_because": "Measured, but the frozen pricing path gives it no eligible price: MachineFi publishes a usage tariff for Trio-Spark (USD 0.042 per 1M billed input tokens, output not billed), yet `trio-spark-v1.3` is in neither frozen tariff set (`AGE_QUALIFIED_API`, `LAUNCH_LIST_PRICE_API`) and the system declares no base model and releases no weights, so there is no base-model market reference either — the frozen module (unchanged on this point since the 32117d75... module it was first checked against) returns \"no eligible 30-day list price or base-model estimate\" (verified by executing it; measurement receipts on file). Without a cost axis there is no composite, so the row shows Intelligence, Calibration and Capability and is not ranked, the same outcome as `sage-1.4.0` and as `subq-snap-preview` before CoS deputy decision #19174. The pricing question is open and can be decided in a later release without re-measuring anything.",
   "noul_decisive": {
    "acc_among_decisive": 0.905829596412556,
    "decisive": 223,
    "decisive_rate": 0.6092896174863388,
    "n": 375,
    "supported": true,
    "valid": 366
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.64268754215436,
     "cost": null,
     "intelligence": 28.409112519894506,
     "speed": 76.46737050345754
    },
    "board_release": "v1.6.43",
    "capability": 57.025900031024435,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.2837795019731857,
     "p50_s_raw": 1.2837795019731857,
     "p95_s_adjusted": 1.7570030387170845,
     "p95_s_raw": 1.7570030387170845
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://github.com/machinefi/trio-spark",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.3097197005117778,
    "p50_s_raw": 1.3097197005117778,
    "p95_s_adjusted": 1.8397685555450152,
    "p95_s_raw": 1.8397685555450152
   },
   "status": {
    "answered_ok": 1398,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 52.81333333333333,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 100.0,
        "score": 8.440000000000003
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 52.81333333333333
      },
      "calibration_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 27.477967258748688,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 28.000000000000004,
        "score": 18.06062500000003
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 27.477967258748688
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 25.287234498413355,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -18.518518518518523,
        "score": 10.640384615384601
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 25.287234498413355
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 72.18962962962964,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 56.56888888888889
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 72.18962962962964
      },
      "jaggedness": {
       "accuracy": 0.4777777777777778,
       "cc": -1.062764445926992,
       "cc_by_type": {
        "choice": 43.69586140519731,
        "noul": -35.13513513513513,
        "score": -11.74901960784316
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.5686900958466453,
       "cc": 45.77148946966644,
       "cc_by_type": {
        "choice": 21.546182970854986,
        "noul": 62.88659793814433,
        "score": 52.8816875
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 45.77148946966644
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -9.75861724758632,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75.0,
        "score": 19.95095238095238
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5309090909090909,
       "cc": 13.281860177476112,
       "cc_by_type": {
        "choice": 55.98304532116076,
        "noul": -33.333333333333336,
        "score": 17.195868544600913
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 13.281860177476112
      },
      "negation_pair_gen": {
       "accuracy": 0.6395348837209303,
       "cc": 21.28888888888888,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": -26.66666666666667,
        "score": 7.199999999999984
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 21.28888888888888
      },
      "option_order": {
       "accuracy": 0.8333333333333334,
       "cc": 78.69507323568575,
       "cc_by_type": {
        "choice": 78.69507323568575
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 78.69507323568575
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 70.99001463001464,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 100.0,
        "score": 46.003076923076925
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.99001463001464
      },
      "usecase": {
       "accuracy": 0.5045454545454545,
       "cc": 21.235606274509806,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": -8.333333333333337,
        "score": 21.059759999999983
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 21.235606274509806
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -25.935775335775343,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -50.0,
        "score": -32.56923076923079
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": -3.255880696364566,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -33.333333333333336,
        "score": 6.515000000000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 57.13679876875722,
       "cc_by_type": {
        "choice": 44.24778761061947,
        "noul": 75.0,
        "score": 52.16260869565217
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 57.13679876875722
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.864742396027307,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.29999999999998
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.864742396027307
      },
      "en": {
       "accuracy": 0.5760208514335361,
       "cc": 28.572488853859696,
       "cc_by_type": {
        "choice": 50.3545233765758,
        "noul": 15.107913669064743,
        "score": 20.25502951593855
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 28.572488853859696
      },
      "es": {
       "accuracy": 0.6451612903225806,
       "cc": 44.747374532194605,
       "cc_by_type": {
        "choice": 51.06035889070147,
        "noul": 75.0,
        "score": 8.181764705882344
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 44.747374532194605
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -66.12021857923497,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": -100.0
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 74.80515995872034,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 66.7684210526316
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 74.80515995872034
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": -20.078258845437613,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -75.0,
        "score": -68.475
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.6,
       "cc": 43.96193779460342,
       "cc_by_type": {
        "choice": 43.88152766952456,
        "noul": 100.0,
        "score": -11.995714285714287
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 43.96193779460342
      },
      "ja": {
       "accuracy": 0.32,
       "cc": -7.710122319301033,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": -40.0,
        "score": 2.651999999999999
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 12.087619047619038,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 36.26285714285713
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 12.087619047619038
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 71.40999999999998,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 97.56333333333333
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 71.40999999999998
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 0.2873246753246785,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 26.83599999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.2873246753246785
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5384615384615384,
       "cc": 13.017297483490639,
       "cc_by_type": {
        "choice": 61.91282268303006,
        "noul": -40.0,
        "score": 17.13906976744186
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 13.017297483490639
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 59.82507758790021,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 32.509999999999984
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 59.82507758790021
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -11.35921985815603,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60.0,
        "score": 28.049999999999997
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": -16.836630434782602,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0.0,
        "score": -85.29249999999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 47.08231182795699,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": -26.495
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 47.08231182795699
      },
      "zh": {
       "accuracy": 0.4,
       "cc": 1.6684934086628982,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": 14.609999999999989
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.6684934086628982
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.608,
       "cc": 49.36757304961825,
       "cc_by_type": {
        "choice": 49.36757304961825
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 49.36757304961825
      },
      "noul": {
       "accuracy": 0.5386666666666666,
       "cc": 7.733333333333325,
       "cc_by_type": {
        "noul": 7.733333333333325
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 7.733333333333325
      },
      "score": {
       "accuracy": 0.5013333333333333,
       "cc": 20.112120938628173,
       "cc_by_type": {
        "score": 20.112120938628173
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 20.112120938628173
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -14.063725490196083,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -100.0,
        "score": -10.191176470588248
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 11.755778078604175,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -1.309615384615359
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 11.755778078604175
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 51.84317073170731,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 54.30999999999997
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.84317073170731
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -33.242417153996115,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100.0,
        "score": -6.160000000000032
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 45.9877324879227,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 48.10812499999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 45.9877324879227
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -60.33445492662474,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -138.86499999999998
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 14.631414924944329,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 6.35923076923075
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 14.631414924944329
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -17.870361561379593,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100.0,
        "score": 30.012608695652165
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -25.707962962962966,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60.0,
        "score": -12.95722222222222
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 21.857077304090158,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 33.33333333333333,
        "score": -3.5222727272727017
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 21.857077304090158
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 5.086647874089732,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -33.333333333333336,
        "score": -28.150909090909092
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 5.086647874089732
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -38.41625896057349,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -85.14125000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 18.304845738252638,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -19.999999999999996,
        "score": 32.496111111111084
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 18.304845738252638
      },
      "model_routing": {
       "accuracy": 0.5345622119815668,
       "cc": 43.44288566468256,
       "cc_by_type": {
        "choice": 8.831168831168855,
        "noul": 78.78787878787878,
        "score": 42.709609375000014
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 43.44288566468256
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": -0.4494520341725945,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 33.20058823529411
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 54.2654650858272,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 46.176153846153845
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.2654650858272
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -38.847736887858595,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100.0,
        "score": -72.20823529411766
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": -14.521700483091797,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -100.0,
        "score": 28.89866666666664
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.7916666666666666,
       "cc": 67.4370359617906,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 50.0,
        "score": 73.66342105263158
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 67.4370359617906
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.49196128193122,
      86.51567270201278
     ],
     "I": [
      22.829086556647436,
      32.828914916616085
     ],
     "capability": [
      52.19692696654943,
      58.86725786769748
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 78
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.64268754215436,
     "capability": 57.025900031024435,
     "intelligence": 28.409112519894506
    },
    "per_type": {
     "choice": {
      "accuracy": 0.608,
      "calibration": 85.40040540540541,
      "cc": 49.36757304961825,
      "cc_by_type": {
       "choice": 49.36757304961825
      },
      "cc_tiered": 49.31269272123022,
      "cc_tiered_open": 48.51140713192591,
      "cc_tiered_sealed": 50.113978310534534,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.36757304961825
     },
     "noul": {
      "accuracy": 0.5386666666666666,
      "calibration": 81.90453551912569,
      "cc": 7.733333333333325,
      "cc_by_type": {
       "noul": 7.733333333333325
      },
      "cc_tiered": 11.649413388543826,
      "cc_tiered_open": 23.29882677708765,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 7.733333333333325
     },
     "score": {
      "accuracy": 0.5013333333333333,
      "calibration": 89.62312170193202,
      "cc": 20.112120938628173,
      "cc_by_type": {
       "score": 20.112120938628173
      },
      "cc_tiered": 24.265231449909475,
      "cc_tiered_open": 31.739283798436347,
      "cc_tiered_sealed": 16.791179101382603,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 20.112120938628173
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.10133333333333339,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.02400000000000002,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.04533333333333334,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Uniphore",
   "axes": {
    "calibration": 79.97905975708014,
    "cost": 64.54050247464266,
    "intelligence": 34.15161183674326,
    "speed": 87.30790070278863
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09746896580594484,
      "ece_score": 80.50620683881104,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.50620683881104,
      "score_before_time_limit": 80.50620683881104,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2815013404825737,
      "brier": 0.17946280164043968,
      "ece": 0.12226593407275449,
      "n": 373,
      "score": 75.5468131854491,
      "score_before_time_limit": 75.5468131854491,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9489247311827957,
      "interval90_width": 0.5589157706093193,
      "n": 372,
      "nrps": 0.11761783656774456,
      "score": 83.88415924698029,
      "score_before_time_limit": 83.88415924698029,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.10234948924632484
     }
    },
    "score": 79.97905975708014
   },
   "capability": 57.0653357969117,
   "composite_ci95": {
    "A": [
     16.384778067728345,
     37.28654558313603
    ],
    "B": [
     13.876698896212954,
     33.41233337397013
    ],
    "C": [
     11.378318102589125,
     25.893434432733358
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: frozen DeepInfra snapshot jevbench-v15-measure-20260925 receipts/DEEPINFRA-MODELS-20260925.json (SHA-256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f); listing Qwen/Qwen3.5-4B (exact base, merge_info.json base_revision 851bf6e8), USD 0.03/M input and USD 0.15/M output. Note (METHOD-v1.5-ADDENDUM-PRICING-INTERPRETATION-1 I-3): the frozen DeepInfra Qwen/Qwen3.5-4B entry is flagged deprecated (replaced_by Qwen/Qwen3.5-9B) while still showing this price; it remains the frozen reference. The OpenRouter /api/v1/models snapshot in source/public-docs (fetched 2026-10-12T07:05:09Z) lists no Qwen3.5-4B route. Self-hosted GPU deployment has no provider tariff, so price_kind is estimate. The native readout is one causal forward pass with no generated tokens: usage.output_tokens is 0 by construction (logits_to_keep=1). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.015205 per 1,000 decisions (0.235x the official cost cap USD 0.064595), against USD 0.015053 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.015052840000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01520488888888889
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Uniphore-4B-Decider",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's inference.py DecisionModel.predict_jev_task unchanged (bf16, sdpa, one forward pass, softmax over single-token option codes, temperatures.json binary T 1.0353 / choice 1.0) behind a first-party loopback TypeSafe shim; transformers 5.17 reference PyTorch kernels (no flash-linear-attention / causal-conv1d in the image) Paid fast-lane order 4b3cb7b8 (public). Merged text-only Qwen3.5-4B LoRA-SFT checkpoint (Qwen3_5ForCausalLM). Runtime vllm/vllm-openai:v0.31.0 image (torch 2.13, transformers 5.17; vLLM unused) instead of the documented torch 2.8 / transformers 5.12 + fla kernels, so latency was measured on the reference PyTorch linear-attention path. Requests over the documented 16,384-token wrapper limit answer HTTP 422 (refused, not truncated). Choice options are presented in the harness's criteria order (labels are not sent), unlike the author's historical label-order evaluator. Output tokens are 0 (no generation). The author discloses that the public JevBench set was inspected during development.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 40.125336699801856,
    "I_sealed": 28.17788697368465,
    "base": 34.15161183674326,
    "excess": 2.107922955761495,
    "gap": 11.947449726117206,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 58.31864130561554,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965713,
       "judge": 64.40342781806197,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 22.505175983436857,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 0.0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 39.55219281035318,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.70208847387271,
       "hard": 38.36737803800065,
       "judge": 36.095989014069396,
       "standard": 39.531180217724135
      }
     },
     "sealed|choice": {
      "cc": 50.50756397649687,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 41.713889210130674,
       "judge": 53.28109696376104,
       "standard": 55.38915840539971
      }
     },
     "sealed|noul": {
      "cc": 11.54997858549068,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 27.586206896551737,
       "hard": 9.090909090909083,
       "judge": -7.692307692307687,
       "standard": 37.31343283582089
      }
     },
     "sealed|score": {
      "cc": 22.4761183590664,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 33.82756717731815,
       "hard": 17.222503109566212,
       "judge": 17.372987543179363,
       "standard": 34.962320672771455
      }
     }
    }
   },
   "jevbench_score": 27.150718036308863,
   "key": "uniphore-4b-decider",
   "last_measured_on": "2026-10-12",
   "licence": "not stated in the model card (no licence field; base Qwen/Qwen3.5-4B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.11",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Uniphore/uniphore-4b-decider@2e22ed63ae0f9de590bbd368f3bfa7c678f78e9d",
   "noul_decisive": {
    "acc_among_decisive": 0.8246268656716418,
    "decisive": 268,
    "decisive_rate": 0.7184986595174263,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.97905975708014,
     "cost": 64.54050247464266,
     "intelligence": 34.15161183674326,
     "speed": 87.70990379280212
    },
    "board_release": "v1.6.43",
    "capability": 57.0653357969117,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 71,
    "capability_rank": 97,
    "composite_ci95": {
     "A": [
      16.39619384096642,
      37.316496815173174
     ],
     "B": [
      13.883211553134624,
      33.43171289678618
     ],
     "C": [
      11.386245722893339,
      25.91423389942581
     ]
    },
    "jevbench_score": 27.171470959871687,
    "measured_in": "v1.6.11",
    "method": "v1.6",
    "open_board_rank": 74,
    "rank": 107,
    "ranks": {
     "A": 107,
     "B": 110,
     "C": 119,
     "capability": 151
    },
    "scores": {
     "A": 27.171470959871687,
     "B": 23.81217493415051,
     "C": 18.86907705546645
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3529104620451108,
     "p50_s_raw": 0.10145523102255538,
     "p95_s_adjusted": 0.4801147931138985,
     "p95_s_raw": 0.16505739655694926
    },
    "views": {
     "Balanced 33:33:33": 24.914288994020374,
     "Emphasis on Accuracy 60:20:20": 20.330194245515276,
     "Emphasis on Cost 20:20:60": 26.761533337866197,
     "Emphasis on Speed 20:60:20": 29.535256858217686,
     "Intelligence only": 15.932855168800014
    }
   },
   "rank": 121,
   "ranked": true,
   "ranks": {
    "A": 121,
    "B": 124,
    "C": 136,
    "capability": 168
   },
   "repo": "https://huggingface.co/Uniphore/uniphore-4b-decider",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 27.150718036308863,
    "B": 23.799421114781396,
    "C": 18.85466530299227
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.36780445810873064,
    "p50_s_raw": 0.10890222905436531,
    "p95_s_adjusted": 0.5053507804637775,
    "p95_s_raw": 0.17767539023188872
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 46.27803208099471,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100.0,
        "score": -19.499237090349197
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 46.27803208099471
      },
      "calibration_gen": {
       "accuracy": 0.43564356435643564,
       "cc": 14.180095745864966,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": 19.999999999999996,
        "score": 2.0736912672873586
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.180095745864966
      },
      "composite_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 14.724448695924453,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -33.333333333333336,
        "score": 10.027004624358726
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 14.724448695924453
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 84.90026374908027,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 54.7007912472408
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.90026374908027
      },
      "jaggedness": {
       "accuracy": 0.37222222222222223,
       "cc": 1.9908303184659708,
       "cc_by_type": {
        "choice": 19.15303176130896,
        "noul": -18.918918918918916,
        "score": 5.738378113007869
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 1.9908303184659708
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 63.6252672621707,
       "cc_by_type": {
        "choice": 63.546711279387154,
        "noul": 69.0721649484536,
        "score": 58.25692555867135
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.6252672621707
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -12.323508173824118,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50.0,
        "score": -0.3725863771424587
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6036363636363636,
       "cc": 30.98148799735982,
       "cc_by_type": {
        "choice": 56.9611998695794,
        "noul": -5.555555555555558,
        "score": 41.53881967805562
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.98148799735982
      },
      "negation_pair_gen": {
       "accuracy": 0.47674418604651164,
       "cc": 2.037509626501965,
       "cc_by_type": {
        "choice": 50.0,
        "noul": -26.66666666666667,
        "score": -17.220804453827434
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 2.037509626501965
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 74.81837004423572,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100.0,
        "score": 60.49114616874321
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.81837004423572
      },
      "usecase": {
       "accuracy": 0.5227272727272727,
       "cc": 25.901229344281592,
       "cc_by_type": {
        "choice": 49.75490196078433,
        "noul": 0.0,
        "score": 27.94878607206045
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 25.901229344281592
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 17.704048175114554,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50.0,
        "score": -20.69737928418014
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 17.704048175114554
      },
      "cs": {
       "accuracy": 0.2857142857142857,
       "cc": 47.368421052631575,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 47.368421052631575
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 17.895602852705036,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 7.757469080388535
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 17.895602852705036
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 69.8201611580944,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 58.35428878401772
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 69.8201611580944
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 15.14372656235736,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 95.896729035303
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 15.14372656235736
      },
      "en": {
       "accuracy": 0.578627280625543,
       "cc": 31.996951914881567,
       "cc_by_type": {
        "choice": 51.2332043787603,
        "noul": 20.863309352517987,
        "score": 23.89434201336642
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.996951914881567
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 70.78955082120461,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 22.156580685473525
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 70.78955082120461
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 41.781260838599806,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 57.58421967426936
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 41.781260838599806
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 40.776530517809675,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 64.6825327298996
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 40.776530517809675
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": -0.0880455013773845,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -50.0,
        "score": 16.77496964111924
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 73.53419736027718,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 20.602592080831517
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.53419736027718
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 28.657352573898077,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 8.863241200645778
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 28.657352573898077
      },
      "ko": {
       "accuracy": 0.4,
       "cc": 4.525100120476314,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 13.575300361428944
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 4.525100120476314
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.43961465493258,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 83.98551063146442
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.43961465493258
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 24.53556347288955,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -33.333333333333336,
        "score": 84.8621016740799
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 24.53556347288955
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 25.142832279070205,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40.0,
        "score": 45.89823869078658
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 25.142832279070205
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 51.72941097740019,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 48.22300016849992
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 51.72941097740019
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 10.795858765536167,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 54.515235871076584
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 10.795858765536167
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 18.82565444321844,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50.0,
        "score": -50.04477580077946
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 18.82565444321844
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.86846162619129,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 53.86344939470291
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.86846162619129
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -6.045504219096912,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -33.95572169683876
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6333333333333333,
       "cc": 52.63973669607149,
       "cc_by_type": {
        "choice": 52.63973669607149
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 52.63973669607149
      },
      "noul": {
       "accuracy": 0.5893333333333334,
       "cc": 17.866666666666674,
       "cc_by_type": {
        "noul": 17.866666666666674
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 17.866666666666674
      },
      "score": {
       "accuracy": 0.5013333333333333,
       "cc": 27.205330671354698,
       "cc_by_type": {
        "score": 27.205330671354698
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.205330671354698
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -9.794702331809445,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -50.0,
        "score": -15.384106995428336
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 14.86522245682505,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 8.018717750047266
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 14.86522245682505
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 15.191320236519909,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0.0,
        "score": -21.905714087188244
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 15.191320236519909
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -14.546261990141502,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": 38.232559058815255
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 2.453084074518673,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -33.333333333333336,
        "score": 27.649107296019793
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 2.453084074518673
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -35.326337874466766,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -66.66666666666667,
        "score": -76.41926519572735
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 57.88596023856343,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 71.42857142857142,
        "score": 31.64107399300122
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.88596023856343
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -1.762092816941351,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50.0,
        "score": 17.884453256493018
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 16.529699165925933,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 32.0890974977778
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 16.529699165925933
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 7.696866416285641,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 0.0,
        "score": 25.874325158921163
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 7.696866416285641
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 29.10220149075188,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 10.045622560110957
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 29.10220149075188
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -26.283415618075534,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -48.742719972506166
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 24.557631257691053,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 39.738152890155696
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 24.557631257691053
      },
      "model_routing": {
       "accuracy": 0.783410138248848,
       "cc": 65.14331381778801,
       "cc_by_type": {
        "choice": 70.7792207792208,
        "noul": 78.78787878787878,
        "score": 45.86284188626449
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 65.14331381778801
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": -0.15637279529120462,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100.0,
        "score": 57.112455510479556
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.82621163125395,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 47.85839348243408
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.82621163125395
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -10.939838270553587,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100.0,
        "score": 11.515460557797375
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 3.5292829705879094,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 1.892196737850682
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 3.5292829705879094
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 73.5250465829234,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50.0,
        "score": 81.25129633240009
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 73.5250465829234
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      74.74001355777733,
      82.37757108771744
     ],
     "I": [
      27.86490456594573,
      38.9213316269163
     ],
     "capability": [
      52.347472852953594,
      59.90975069911703
     ],
     "composite": {
      "A": [
       16.384778067728345,
       37.28654558313603
      ],
      "B": [
       13.876698896212954,
       33.41233337397013
      ],
      "C": [
       11.378318102589125,
       25.893434432733358
      ]
     },
     "seed": 79
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.97905975708014,
     "capability": 57.0653357969117,
     "intelligence": 34.15161183674326
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6333333333333333,
      "calibration": 80.50620683881104,
      "cc": 52.63973669607149,
      "cc_by_type": {
       "choice": 52.63973669607149
      },
      "cc_tiered": 54.413102641056206,
      "cc_tiered_open": 58.31864130561554,
      "cc_tiered_sealed": 50.50756397649687,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 52.63973669607149
     },
     "noul": {
      "accuracy": 0.5893333333333334,
      "calibration": 75.5468131854491,
      "cc": 17.866666666666674,
      "cc_by_type": {
       "noul": 17.866666666666674
      },
      "cc_tiered": 17.027577284463767,
      "cc_tiered_open": 22.505175983436857,
      "cc_tiered_sealed": 11.54997858549068,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 17.866666666666674
     },
     "score": {
      "accuracy": 0.5013333333333333,
      "calibration": 83.88415924698029,
      "cc": 27.205330671354698,
      "cc_by_type": {
       "score": 27.205330671354698
      },
      "cc_tiered": 31.01415558470979,
      "cc_tiered_open": 39.55219281035318,
      "cc_tiered_sealed": 22.4761183590664,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.205330671354698
     }
    },
    "raw_on_A": {
     "calibration": 81.83637520750273,
     "capability": 58.75237966723043,
     "intelligence": 35.66838412695813
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 24.89102871623233,
    "Emphasis on Accuracy 60:20:20": 20.32089689500326,
    "Emphasis on Cost 20:20:60": 26.74542557194252,
    "Emphasis on Speed 20:60:20": 29.476479181130028,
    "Intelligence only": 15.932855168800014
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 66.9335611044975,
    "cost": 79.21519751338194,
    "intelligence": 5.342898619124496,
    "speed": 73.05648557202497
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16337254738540735,
      "ece_score": 67.32549052291853,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 66.42781731594629,
      "score_before_time_limit": 67.32549052291853,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.8053333333333333,
      "brier": 0.28665151355378216,
      "ece": 0.18020482243198754,
      "n": 375,
      "score": 63.61792065752995,
      "score_before_time_limit": 63.959035513602494,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.8106666666666666,
      "interval90_width": 0.5722222222222216,
      "n": 375,
      "nrps": 0.18890773137678013,
      "score": 70.75494534001628,
      "score_before_time_limit": 71.32554973791963,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.1922906369324137
     }
    },
    "score": 66.9335611044975
   },
   "capability": 36.138229861811,
   "composite_ci95": {
    "A": [
     0.051992232969144535,
     0.488734028804126
    ],
    "B": [
     0.03463221996897877,
     0.3454691214907552
    ],
    "C": [
     0.0361057173396837,
     0.33939863111397645
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.) through the frozen pricing_v15 path (module sha256 6202ef78...): vega-U is a 14.3 M decision engine on a FROZEN Qwen/Qwen3.5-0.8B reader, which resolves to deepinfra:Qwen/Qwen3.5-0.8B in the dated 25 Sep DeepInfra catalog snapshot (sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f), USD 0.01/M input and USD 0.05/M output - the same basis and the same numbers the live vegaml-0.8b row carries. The model generates no text: 0 output tokens on every item, so cost = input tokens x USD 0.01/M. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.004930 per 1,000 decisions (0.076x the official cost cap USD 0.064595), against USD 0.012253 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.012253273333333332
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0049297710437710435
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega-U (Qwen3.5-0.8B reader)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-18",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "our own CPU server (EPYC 7502P, 32 of 64 threads, 96 GB), disposable docker container with --network none (cut proven from inside before the sealed mount), --cap-drop=ALL, no-new-privileges, unprivileged user, one read-only sealed input mount; reviewed vegaml==0.12.2, torch 2.14.1+cpu, transformers 5.19.0; fp32 throughout (the author's own backbone_dtype() choice on CPU); context 73,728, and a longer state is cut on the right by the author's own build_prompt - 15 of the 1,500 items sit at that cap and are answered on a truncated state. Reference PyTorch paths only.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.448296966828988,
    "I_sealed": 4.237500271420004,
    "base": 5.342898619124496,
    "excess": -7.628730074946727,
    "gap": 2.210796695408984,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 19.344890900486963,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486495,
       "hard": 9.405473419314257,
       "judge": 16.94133157547792,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -82.6086956521739,
       "judge": -70.0,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 23.713101067178897,
       "hard": -16.932168774632817,
       "judge": -31.207540516067354,
       "standard": -9.704346143973197
      }
     },
     "sealed|choice": {
      "cc": 12.71250081426001,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 18.04320203303686,
       "hard": 11.009598704753081,
       "judge": 14.495592556317357,
       "standard": 10.778316810799424
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -72.41379310344827,
       "hard": -83.63636363636364,
       "judge": -81.53846153846153,
       "standard": -82.08955223880598
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 1.711176163732453,
       "hard": -8.801799755235983,
       "judge": -15.186794142496286,
       "standard": -9.891030118368649
      }
     }
    }
   },
   "jevbench_score": 0.1999615926282204,
   "key": "vega-u-qwen3.5-0.8b",
   "last_measured_on": "2026-10-18",
   "licence": "vega-U checkpoint apache-2.0 (card and metadata agree); vegaml package apache-2.0; frozen reader Qwen/Qwen3.5-0.8B apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.29",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-U@b3e7383e463ef3b78324408cd82f4ab0d8f2e75a",
   "noul_decisive": {
    "acc_among_decisive": 0.5342465753424658,
    "decisive": 73,
    "decisive_rate": 0.19571045576407506,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 67.53669192481355,
     "cost": 79.21519751338194,
     "intelligence": 5.503418464051958,
     "speed": 73.8348503635803
    },
    "board_release": "v1.6.43",
    "capability": 36.52005519443276,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.059047982307085994,
      0.5209750966462322
     ],
     "B": [
      0.039373315414271894,
      0.3690362057340607
     ],
     "C": [
      0.04100554326880973,
      0.3617882615598834
     ]
    },
    "jevbench_score": 0.21762292373052544,
    "measured_in": "v1.6.29",
    "method": "v1.6",
    "open_board_rank": 169,
    "rank": 213,
    "ranks": {
     "A": 213,
     "B": 213,
     "C": 213,
     "capability": 213
    },
    "scores": {
     "A": 0.21762292373052544,
     "B": 0.14979599675011504,
     "C": 0.15112703036842048
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.5407199749955907,
     "p50_s_raw": 0.6953599874977954,
     "p95_s_adjusted": 2.684054008999373,
     "p95_s_raw": 1.2670270044996865
    },
    "views": {
     "Balanced 33:33:33": 0.1748431414819662,
     "Emphasis on Accuracy 60:20:20": 0.10603360540342595,
     "Emphasis on Cost 20:20:60": 0.25984513652167707,
     "Emphasis on Speed 20:60:20": 0.2578104829204347,
     "Intelligence only": 0.06667416738798192
    }
   },
   "rank": 236,
   "ranked": true,
   "ranks": {
    "A": 236,
    "B": 236,
    "C": 236,
    "capability": 236
   },
   "repo": "https://huggingface.co/nandakishorm/vega-U",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.1999615926282204,
    "B": 0.13738157150274738,
    "C": 0.13886221710293087
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.6106067149725276,
    "p50_s_raw": 0.7303033574862638,
    "p95_s_adjusted": 3.071580751580768,
    "p95_s_raw": 1.460790375790384
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.3181818181818182,
       "cc": -1.8642060422197488,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": 0.0,
        "score": -13.92595145999258
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.2079207920792079,
       "cc": -19.789838255276152,
       "cc_by_type": {
        "choice": -8.165429480381752,
        "noul": -43.99999999999999,
        "score": -7.204085285446715
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.1485148514851485,
       "cc": -44.963648001333546,
       "cc_by_type": {
        "choice": -7.317073170731707,
        "noul": -100.0,
        "score": -27.57387083326892
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.3125,
       "cc": -18.313256734755296,
       "cc_by_type": {
        "choice": 14.285714285714292,
        "noul": -60.0,
        "score": -9.225484489980174
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0.0
      },
      "jaggedness": {
       "accuracy": 0.2111111111111111,
       "cc": -34.81620625476442,
       "cc_by_type": {
        "choice": -3.9461020211741884,
        "noul": -89.1891891891892,
        "score": -11.313327553929863
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.22364217252396165,
       "cc": -24.720816695072543,
       "cc_by_type": {
        "choice": 17.583868979484027,
        "noul": -85.56701030927836,
        "score": -6.179308755423296
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0.0
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -19.02278279739924,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -75.0,
        "score": -20.212678289104936
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.3236363636363636,
       "cc": -22.568131932211916,
       "cc_by_type": {
        "choice": 23.703945223345297,
        "noul": -77.77777777777779,
        "score": -13.630563242203264
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.23255813953488372,
       "cc": -32.29973265796035,
       "cc_by_type": {
        "choice": 3.333333333333336,
        "noul": -80.0,
        "score": -20.232531307214387
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.3333333333333333,
       "cc": 14.780292942743008,
       "cc_by_type": {
        "choice": 14.780292942743008
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 14.780292942743008
      },
      "perturb_pair": {
       "accuracy": 0.40625,
       "cc": -12.512300010905099,
       "cc_by_type": {
        "choice": 27.92792792792793,
        "noul": -60.0,
        "score": -5.464827960643226
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0.0
      },
      "usecase": {
       "accuracy": 0.2909090909090909,
       "cc": -18.881107912882545,
       "cc_by_type": {
        "choice": 22.794117647058847,
        "noul": -83.33333333333334,
        "score": 3.8958919476268616
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -36.55525876106678,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -100.0,
        "score": -14.4276810451051
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.15384615384615385,
       "cc": -65.52162213002279,
       "cc_by_type": {
        "choice": -3.6866359447004595,
        "noul": -100.0,
        "score": -92.87823044536792
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.40625,
       "cc": -18.671575997460423,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": -100.0,
        "score": -24.156320912735254
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "el": {
       "accuracy": 0.26666666666666666,
       "cc": -32.16558414122873,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -100.0,
        "score": -29.45764627843479
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.24413553431798435,
       "cc": -26.01741064970663,
       "cc_by_type": {
        "choice": 9.49585677499659,
        "noul": -79.13669064748201,
        "score": -8.41139807663447
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.3225806451612903,
       "cc": -23.899883923976507,
       "cc_by_type": {
        "choice": 11.908646003262646,
        "noul": -75.0,
        "score": -8.608297775192163
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -40.6825249989779,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100.0,
        "score": -56.47380450513042
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": -11.687923076823088,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": -100.0,
        "score": 28.465642534236622
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": -3.1055582694130988,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -75.0,
        "score": 15.96265480069925
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.4,
       "cc": 5.813112671547951,
       "cc_by_type": {
        "choice": 29.851909586905684,
        "noul": -33.333333333333336,
        "score": 20.920761761071503
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 5.813112671547951
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 7.3870916359845324,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": -0.6504804534443354
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 7.3870916359845324
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -51.878016279386806,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0,
        "score": -22.300715504827085
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.35714285714285715,
       "cc": 5.358585021957109,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -33.333333333333336,
        "score": 16.075755065871324
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 5.358585021957109
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -19.24038798812712,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": -5.773112016329418
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.23076923076923078,
       "cc": -30.678147730963047,
       "cc_by_type": {
        "choice": 16.208209902666106,
        "noul": -80.0,
        "score": -28.242653095555248
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.35714285714285715,
       "cc": -8.753322219103685,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": -60.0,
        "score": 7.988707473508039
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "sv": {
       "accuracy": 0.0,
       "cc": -39.34001974615732,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -100.0,
        "score": 9.63951522961315
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -52.30606124559736,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100.0,
        "score": -91.70079243244427
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -68.27594716841715,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100.0,
        "score": -108.05364795686434
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.1,
       "cc": -38.962930251919914,
       "cc_by_type": {
        "choice": -27.11864406779661,
        "noul": -100.0,
        "score": 10.229853312036862
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.328,
       "cc": 13.201553799345579,
       "cc_by_type": {
        "choice": 13.201553799345579
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.201553799345579
      },
      "noul": {
       "accuracy": 0.104,
       "cc": -79.2,
       "cc_by_type": {
        "noul": -79.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.2853333333333333,
       "cc": -10.082003664821503,
       "cc_by_type": {
        "score": -10.082003664821503
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -34.926263523751715,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -100.0,
        "score": -24.778790571255136
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.20833333333333334,
       "cc": -24.119846467833653,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -100.0,
        "score": 2.1746220871822763
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 20.313465191128632,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 0.0,
        "score": 25.98104597988996
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 20.313465191128632
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -32.196963481052016,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100.0,
        "score": -3.0236389811677533
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.23529411764705882,
       "cc": -45.51840486299776,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100.0,
        "score": -49.59869284986284
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -46.93173196122063,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -98.65683110378765
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.3333333333333333,
       "cc": -23.154503740343333,
       "cc_by_type": {
        "choice": 31.372549019607845,
        "noul": -100.0,
        "score": -0.8360602406378481
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -46.359310004542714,
       "cc_by_type": {
        "choice": -4.529616724738673,
        "noul": -100.0,
        "score": -34.54831328888945
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -17.751149481419645,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100.0,
        "score": 30.079884889074393
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -27.831068186330523,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -100.0,
        "score": 19.290521351072666
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.28,
       "cc": -11.984104879552689,
       "cc_by_type": {
        "choice": 30.232558139534877,
        "noul": -100.0,
        "score": 33.81512722180705
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -35.612368484434,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100.0,
        "score": -74.57904093717296
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 5.575609350286421,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -60.0,
        "score": 11.275772388671168
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 5.575609350286421
      },
      "model_routing": {
       "accuracy": 0.2534562211981567,
       "cc": -21.434671725606155,
       "cc_by_type": {
        "choice": 25.194805194805216,
        "noul": -78.78787878787878,
        "score": -10.710941583744905
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0.0
      },
      "moderation_trust_safety": {
       "accuracy": 0.2777777777777778,
       "cc": -35.69443881183727,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -100.0,
        "score": -26.469112980617382
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.2222222222222222,
       "cc": -30.652547782563005,
       "cc_by_type": {
        "choice": -0.2785515320334252,
        "noul": -100.0,
        "score": 8.320908184344411
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 11.928697621534182,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 0.0,
        "score": -5.10060664278662
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 11.928697621534182
      },
      "scientific_discovery": {
       "accuracy": 0.21739130434782608,
       "cc": -26.799833636121605,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -66.66666666666667,
        "score": -12.283558879379308
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.20833333333333334,
       "cc": -28.36783840626855,
       "cc_by_type": {
        "choice": 25.26690391459075,
        "noul": -100.0,
        "score": -10.370419133396403
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      60.99693003197813,
      70.28216529528869
     ],
     "I": [
      3.33432352068163,
      7.3529305629723165
     ],
     "capability": [
      32.78281449779358,
      38.263101265133066
     ],
     "composite": {
      "A": [
       0.051992232969144535,
       0.488734028804126
      ],
      "B": [
       0.03463221996897877,
       0.3454691214907552
      ],
      "C": [
       0.0361057173396837,
       0.33939863111397645
      ]
     },
     "seed": 80
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.9335611044975,
     "capability": 36.138229861811,
     "intelligence": 5.342898619124496
    },
    "per_type": {
     "choice": {
      "accuracy": 0.328,
      "calibration": 66.42781731594629,
      "cc": 13.201553799345579,
      "cc_by_type": {
       "choice": 13.201553799345579
      },
      "cc_tiered": 16.028695857373485,
      "cc_tiered_open": 19.344890900486963,
      "cc_tiered_sealed": 12.71250081426001,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.201553799345579
     },
     "noul": {
      "accuracy": 0.104,
      "calibration": 63.61792065752995,
      "cc": -79.2,
      "cc_by_type": {
       "noul": -79.2
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2853333333333333,
      "calibration": 70.75494534001628,
      "cc": -10.082003664821503,
      "cc_by_type": {
       "score": -10.082003664821503
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 66.70020516657101,
     "capability": 35.870650331003844,
     "intelligence": 5.0410954954366805
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.1604669680317333,
    "Emphasis on Accuracy 60:20:20": 0.09712939875705864,
    "Emphasis on Cost 20:20:60": 0.23915969716756003,
    "Emphasis on Speed 20:60:20": 0.23704625758714523,
    "Intelligence only": 0.061008562485919474
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 76.81388603317517,
    "cost": 64.90155987179207,
    "intelligence": 8.45673307172142,
    "speed": 88.8199586422866
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07314029135902292,
      "ece_score": 85.37194172819542,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 84.2336491718195,
      "score_before_time_limit": 85.37194172819542,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.6693333333333333,
      "brier": 0.22771527118772983,
      "ece": 0.12364099316545366,
      "n": 375,
      "score": 74.8703517596191,
      "score_before_time_limit": 75.27180136690927,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.768,
      "interval90_width": 0.4584444444444443,
      "n": 375,
      "nrps": 0.1744417912695621,
      "score": 71.33765716808692,
      "score_before_time_limit": 71.91296085492633,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.1936494958159556
     }
    },
    "score": 76.81388603317517
   },
   "capability": 42.63530955244829,
   "composite_ci95": {
    "A": [
     0.36273978490149095,
     1.7366276349061507
    ],
    "B": [
     0.2528709513330781,
     1.288977371078567
    ],
    "C": [
     0.25190262840381317,
     1.2059914131292713
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.) through the frozen pricing_v15 path (module sha256 aa2afcd5...): vega-U is a 14.3 M decision engine on a FROZEN Qwen/Qwen3.5-4B reader, which resolves to deepinfra:Qwen/Qwen3.5-4B in the dated 25 Sep DeepInfra catalog snapshot (sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f), USD 0.03/M input and USD 0.15/M output. The model generates no text: 0 output tokens on every item, so cost = input tokens x USD 0.03/M. COMMON COST BASIS: published cost is recomputed over the same remaining 1,485 items as every other row; all 1,500 items count for Intelligence. ALL-ITEMS DIAGNOSTIC: USD 0.0367598 per 1,000 over all 1,500 items (0.569x cap).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03675982
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014789313131313127
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega-U (Qwen3.5-4B reader)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment, measured 23 Oct 2026 06:56-07:03 UTC. The declared reviewed loopback shim `python3 vega_shim.py <pinned snapshot>/ckpt <port> vega-u-qwen3.5-4b`, with `VEGA_DEVICE=cuda`, vegaml==0.12.2, torch 2.14.1+cu126, mode=engine, adapters=auto, views=1, no max_len override (73,728); vegaml chooses the GPU dtype. The author's GPU configuration is used; the evaluator's RTX 6000 Ada is the only hardware deviation. The offline container-A2 recount matches the forecast fixed before the run: 1,500 answered, 0 refused, 1,837,991 input tokens, no output tokens. Fifteen items are cut on the right at the author's 73,728-token limit and take 13.8-14.3 s each on this card, so they count as failed under the 10 s rule. Speed is a dedicated-GPU figure measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01. The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 7.801204167922121,
    "I_sealed": 9.11226197552072,
    "base": 8.45673307172142,
    "excess": -11.15058457795431,
    "gap": -1.3110578075985995,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 23.403612503766364,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.378378378378383,
       "hard": 28.279333123623783,
       "judge": 9.030982201713911,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -30.434782608695656,
       "judge": -50.0,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.312243229154305,
       "hard": -8.801569670026566,
       "judge": 0.2849655036505916,
       "standard": -17.412006597581488
      }
     },
     "sealed|choice": {
      "cc": 27.336785926562165,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 29.47903430749683,
       "hard": 24.540302995258457,
       "judge": 26.836434867776703,
       "standard": 32.609154186880424
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -41.379310344827594,
       "hard": -50.90909090909091,
       "judge": -69.23076923076923,
       "standard": -46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 20.512927693345652,
       "hard": 5.426302008135641,
       "judge": -15.699545487050193,
       "standard": -2.3208485625247466
      }
     }
    }
   },
   "jevbench_score": 0.7245180783982694,
   "key": "vega-u-qwen3.5-4b",
   "last_measured_on": "2026-10-23",
   "licence": "vega-U checkpoint apache-2.0 (card and metadata agree); vegaml package apache-2.0; frozen reader Qwen/Qwen3.5-4B apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-U@b3e7383e463ef3b78324408cd82f4ab0d8f2e75a",
   "noul_decisive": {
    "acc_among_decisive": 0.7580645161290323,
    "decisive": 124,
    "decisive_rate": 0.3324396782841823,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.8233873834119,
     "cost": 64.90155987179207,
     "intelligence": 8.801638441109716,
     "speed": 60.651933418279626
    },
    "board_release": "v1.6.43",
    "capability": 44.31251291226081,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.4134795884464807,
      1.7993673499327525
     ],
     "B": [
      0.29304131966029756,
      1.366981924419596
     ],
     "C": [
      0.28713860308783384,
      1.2495606596755224
     ]
    },
    "jevbench_score": 0.7843042690267673,
    "measured_in": "v1.6.32",
    "method": "v1.6",
    "open_board_rank": 156,
    "rank": 199,
    "ranks": {
     "A": 199,
     "B": 199,
     "C": 199,
     "capability": 189
    },
    "scores": {
     "A": 0.7843042690267673,
     "B": 0.570350285841166,
     "C": 0.5446557423796995
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 6.766180534940213,
     "p50_s_raw": 3.3080902674701065,
     "p95_s_adjusted": 12.719296073674922,
     "p95_s_raw": 6.284648036837461
    },
    "views": {
     "Balanced 33:33:33": 0.638871364050114,
     "Emphasis on Accuracy 60:20:20": 0.41567103323551446,
     "Emphasis on Cost 20:20:60": 0.8786972307100303,
     "Emphasis on Speed 20:60:20": 0.8680676350289552,
     "Intelligence only": 0.27274108540540976
    }
   },
   "rank": 224,
   "ranked": true,
   "ranks": {
    "A": 224,
    "B": 224,
    "C": 224,
    "capability": 221
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012. Fifteen items reach the author's 73,728-token context limit and are answered on a right-truncated state; they take 13.8-14.3 s each on this GPU and count as failed under the 10 s rule.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/nandakishorm/vega-U",
   "scores": {
    "A": 0.7245180783982694,
    "B": 0.5178907226942993,
    "C": 0.5031375544432425
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.33023520699989606,
    "p50_s_raw": 0.09011760349994802,
    "p95_s_adjusted": 0.39735690419992126,
    "p95_s_raw": 0.12367845209996063
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.45454545454545453,
       "cc": -25.03596198376302,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -50.0,
        "score": -58.441219284622406
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.27722772277227725,
       "cc": -14.828368777600858,
       "cc_by_type": {
        "choice": 4.559915164369044,
        "noul": -36.0,
        "score": -13.045021497171616
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.31683168316831684,
       "cc": -14.0069374233143,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -40.74074074074075,
        "score": -3.719095919446058
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.7083333333333334,
       "cc": 51.26191529469682,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 20.452412550757128
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 51.26191529469682
      },
      "jaggedness": {
       "accuracy": 0.2611111111111111,
       "cc": -23.663861725959578,
       "cc_by_type": {
        "choice": 4.716073147256991,
        "noul": -62.16216216216216,
        "score": -13.545496162973558
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.38338658146964855,
       "cc": 0.4762587666438864,
       "cc_by_type": {
        "choice": 27.093422558774332,
        "noul": -42.26804123711341,
        "score": 16.603394978270735
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0.4762587666438864
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -53.162603064709884,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100.0,
        "score": -72.88987104979974
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.43636363636363634,
       "cc": -1.4153482680333316,
       "cc_by_type": {
        "choice": 48.15780893381155,
        "noul": -63.888888888888886,
        "score": 11.48503515097734
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.26744186046511625,
       "cc": -22.098041025583615,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": -40.0,
        "score": -19.627456410084186
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.3125,
       "cc": 12.117177097203731,
       "cc_by_type": {
        "choice": 12.117177097203731
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 12.117177097203731
      },
      "perturb_pair": {
       "accuracy": 0.46875,
       "cc": -6.191039931210905,
       "cc_by_type": {
        "choice": 39.93993993993994,
        "noul": -80.0,
        "score": 21.486940266427347
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0.0
      },
      "usecase": {
       "accuracy": 0.40454545454545454,
       "cc": -4.150913529523027,
       "cc_by_type": {
        "choice": 43.627450980392176,
        "noul": -62.5,
        "score": 6.419808431038742
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -55.1179221238957,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50.0,
        "score": -101.06805208597281
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": -16.347518774147833,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -100.0,
        "score": -28.305229133503417
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.46875,
       "cc": 9.53194357801521,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": -50.0,
        "score": 10.454237813691647
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 9.53194357801521
      },
      "el": {
       "accuracy": 0.2,
       "cc": -48.12894813495393,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100.0,
        "score": -60.58796172329757
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.0
      },
      "en": {
       "accuracy": 0.36055603822762816,
       "cc": -7.544666244633066,
       "cc_by_type": {
        "choice": 22.0170610561257,
        "noul": -45.32374100719424,
        "score": 0.6726812171693464
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.45161290322580644,
       "cc": -7.245856233278336,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -25.0,
        "score": -28.222071146817072
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -31.446955625689117,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": 4.019788860637563
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 7.9862271549963095,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": -100.0,
        "score": 45.13515205322422
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 7.9862271549963095
      },
      "hi": {
       "accuracy": 0.42857142857142855,
       "cc": -5.873615988487875,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100.0,
        "score": 32.65848164347492
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.6,
       "cc": -12.368119435536334,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": -100.0,
        "score": -9.045122141371277
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ja": {
       "accuracy": 0.44,
       "cc": 2.520638957463568,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": 0.0,
        "score": 7.64134498994432
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 2.520638957463568
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -38.53246573373655,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": -26.708508312320767
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.5714285714285714,
       "cc": 42.35289813052079,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 33.33333333333333,
        "score": 27.058694391562376
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 42.35289813052079
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -11.660456122154708,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": -9.007342392438167
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.4358974358974359,
       "cc": 9.515495097600082,
       "cc_by_type": {
        "choice": 54.29538721963606,
        "noul": -60.0,
        "score": 34.25109807316419
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 9.515495097600082
      },
      "pt": {
       "accuracy": 0.39285714285714285,
       "cc": -5.205980171443635,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": -60.0,
        "score": 8.023780169228322
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 2.6759101210372713,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60.0,
        "score": 44.62347504396287
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 2.6759101210372713
      },
      "tr": {
       "accuracy": 0.23076923076923078,
       "cc": -36.08796813124616,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100.0,
        "score": -43.04651308939067
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": -27.496009583018008,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100.0,
        "score": -50.229964232924985
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.4,
       "cc": 9.62711250017331,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": 38.48585727453122
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.62711250017331
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.43066666666666664,
       "cc": 26.462427524445555,
       "cc_by_type": {
        "choice": 26.462427524445555
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 26.462427524445555
      },
      "noul": {
       "accuracy": 0.25066666666666665,
       "cc": -49.86666666666667,
       "cc_by_type": {
        "noul": -49.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.368,
       "cc": 0.9290025644768996,
       "cc_by_type": {
        "score": 0.9290025644768996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.9290025644768996
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -19.783631212480014,
       "cc_by_type": {
        "choice": 20.0,
        "noul": -75.0,
        "score": -4.350893637440034
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -24.99740795770514,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -100.0,
        "score": -37.7251431277738
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 16.845304659485688,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 0.0,
        "score": 15.576564384961122
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 16.845304659485688
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -42.72991178726473,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -58.014296765302944
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": -9.175745041749694,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": -65.20839454553895
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -37.19985828253814,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -69.4612100677402
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": -8.951088980880352,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": -6.4050876709323745
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -20.737580599805494,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100.0,
        "score": 10.9579899079006
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -16.26424603536173,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100.0,
        "score": 34.54059522724814
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.2727272727272727,
       "cc": -15.584461807455959,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -66.66666666666667,
        "score": 22.697007154363035
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": -3.5346979404213195,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -100.0,
        "score": 24.279627108968594
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -43.23093560590408,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100.0,
        "score": -32.91861326932515
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 6.361974315412698,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -60.0,
        "score": 2.1185525047793363
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 6.361974315412698
      },
      "model_routing": {
       "accuracy": 0.33640552995391704,
       "cc": -6.464313948023082,
       "cc_by_type": {
        "choice": 25.194805194805216,
        "noul": -42.42424242424242,
        "score": -2.163504614632039
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0.0
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -12.23020819041083,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100.0,
        "score": 32.40726410439131
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.2777777777777778,
       "cc": -1.8346473851051428,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -33.333333333333336,
        "score": 11.394850788045762
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -15.876737884882187,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -100.0,
        "score": 26.261411961117
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": -10.815448509295445,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": -26.649244078610977
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.625,
       "cc": 27.35102015958954,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": -50.0,
        "score": 42.729217062398526
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 27.35102015958954
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.81973083659956,
      79.10365039256962
     ],
     "I": [
      6.5873945818039585,
      11.663246480964204
     ],
     "capability": [
      39.54192140892927,
      44.55782071854231
     ],
     "composite": {
      "A": [
       0.36273978490149095,
       1.7366276349061507
      ],
      "B": [
       0.2528709513330781,
       1.288977371078567
      ],
      "C": [
       0.25190262840381317,
       1.2059914131292713
      ]
     },
     "seed": 81
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 76.81388603317517,
     "capability": 42.63530955244829,
     "intelligence": 8.45673307172142
    },
    "per_type": {
     "choice": {
      "accuracy": 0.43066666666666664,
      "calibration": 84.2336491718195,
      "cc": 26.462427524445555,
      "cc_by_type": {
       "choice": 26.462427524445555
      },
      "cc_tiered": 25.370199215164263,
      "cc_tiered_open": 23.403612503766364,
      "cc_tiered_sealed": 27.336785926562165,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 26.462427524445555
     },
     "noul": {
      "accuracy": 0.25066666666666665,
      "calibration": 74.8703517596191,
      "cc": -49.86666666666667,
      "cc_by_type": {
       "noul": -49.86666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.368,
      "calibration": 71.33765716808692,
      "cc": 0.9290025644768996,
      "cc_by_type": {
       "score": 0.9290025644768996
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.9290025644768996
     }
    },
    "raw_on_A": {
     "calibration": 75.03038816750606,
     "capability": 42.19031445358924,
     "intelligence": 9.350240739672419
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.5922038014681112,
    "Emphasis on Accuracy 60:20:20": 0.3750067344536265,
    "Emphasis on Cost 20:20:60": 0.8139270671824799,
    "Emphasis on Speed 20:60:20": 0.8542674977714012,
    "Intelligence only": 0.24191781959574568
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Victor Bennett (Hugging Face victorbennett; benchmarkheaven.com/submit ref E59F3C77)",
   "axes": {
    "calibration": 83.75263461013449,
    "cost": 62.17924458049355,
    "intelligence": 35.44491489257096,
    "speed": 90.21057479487555
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07788559948821713,
      "ece_score": 84.42288010235657,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.42288010235657,
      "score_before_time_limit": 84.42288010235657,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.34584450402144773,
      "brier": 0.16549543694752686,
      "ece": 0.10689071429896141,
      "n": 373,
      "score": 78.62185714020772,
      "score_before_time_limit": 78.62185714020772,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9516129032258065,
      "interval90_width": 0.603942652329749,
      "n": 372,
      "nrps": 0.10859985038392947,
      "score": 88.21316658783921,
      "score_before_time_limit": 88.21316658783921,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06356840892964317
     }
    },
    "score": 83.75263461013449
   },
   "capability": 59.59877475135272,
   "composite_ci95": {
    "A": [
     23.29437819037973,
     37.91142806589149
    ],
    "B": [
     20.216299228222162,
     34.03666918233901
    ],
    "C": [
     16.17665152109704,
     26.32738060131354
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.) through the frozen pricing_v15 path: base-model references unchanged since sha256 dcfcecb0...2835 (the later re-freezes only add or re-price hosted-API launch-list keys; this row's reference is the same one it was first priced with). The base model Qwen/Qwen3.5-4B carries an EXACT base-model market reference in the frozen 25 Sep catalog - deepinfra:Qwen/Qwen3.5-4B (snapshot sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f) - so resolve() returns rankable:true with a labelled ESTIMATE and no new pricing line is needed. The operator tariff is excluded by the 30-day rule, and the reference itself is flagged in the module as deprecated after the cut-off (replaced by Qwen/Qwen3.5-9B): re-score if it changes. Output tokens are a literal 0 for every item - the model generates no text at all, and the author's own server reports usage.output_tokens: 0 (the pjev-2b convention and the 9 Oct 2026 rule for native option-probability models) - so the output term is 0.0, not 0.15. Multiplied by this row's own measured input tokens (902,188 input tokens; cost per 1,000 decisions over the common 1,485-item basis). Free weights are never cost 100 and never cost 0. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.018226 per 1,000 decisions (0.282x the official cost cap USD 0.064595), against USD 0.018044 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.018043760000000002
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.018226020202020205
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Verdin-4B (Victor Bennett, Qwen3.5-4B + merged LoRA r16 + pointer head)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment. The author's own Verdin engine and server, unmodified, at their declared GPU configuration: `python3 -m verdin.server --model <pinned snapshot> --host 127.0.0.1 --port <p> --backend cuda --dtype bf16 --name verdin-4b` (the author's CUDA bf16 path; fast gated-delta kernels, causal-conv1d, prefill chunk 8192, max sweep 16,384 tokens / 256 rows, server batching defaults), without the earlier CPU run's float32 and `--max-batch 1` (both forbidden by the gate and absent from the launched argv/env). Stack: Python 3.12 venv, torch 2.8.0+cu128 and the other pins of the author's own requirements-cuda.txt (a packaging change only: the author's pinned numpy needs Python 3.12, which the earlier pinned base image did not ship; weights and author code are unchanged and identical to the first use). The declared hardware of the author is one RTX 5090; the evaluator's GPU is an RTX 6000 Ada, the only deviation from the declared configuration. The author's renderer refuses a state over its own 65,536-token cap with HTTP 422 rather than truncating, so the draw's 15 longest items are honest refusals scored as wrong (the same 15 items as in the CPU row); the offline recount matched the pre-registered forecast exactly (1,485 answered, 15 refused, 902,188 input tokens). Speed is a dedicated-GPU figure, measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01, torn down after the run (22 Oct 2026). The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven by three failed outbound probes before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 48.98952596706263,
    "I_sealed": 25.859907489946018,
    "base": 37.42471672850432,
    "excess": 13.290091706760904,
    "gap": 23.129618477116615,
    "penalty": 0.947099082932391,
    "per_type_split": {
     "open|choice": {
      "cc": 69.12490790461862,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 37.862663906142174,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 56.52173913043479,
       "judge": -9.999999999999998,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 39.981006090427115,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.24734845438208,
       "hard": 42.544026794962654,
       "judge": 42.50635554607598,
       "standard": 27.933769315905277
      }
     },
     "sealed|choice": {
      "cc": 53.08980922463883,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 47.958829651902384,
       "judge": 49.75514201762979,
       "standard": 57.28749209027631
      }
     },
     "sealed|noul": {
      "cc": 0.8830767791138365,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 20.68965517241379,
       "hard": -3.6363636363636376,
       "judge": -19.999999999999996,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 23.606836466085404,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 48.65018364828838,
       "hard": 16.741050238290057,
       "judge": 13.688946355597997,
       "standard": 39.69357049630571
      }
     }
    }
   },
   "jevbench_score": 29.859394724952654,
   "key": "verdin-4b",
   "last_measured_on": "2026-10-22",
   "licence": "Weights CC-BY-NC-4.0 (non-commercial; the repo's NOTICE and the model card state that part of the training mix - BeaverTails, the ANLI translations, RAGTruth's MS MARCO and Yelp-derived contexts - is non-commercial). Base model Qwen/Qwen3.5-4B Apache-2.0 (LICENSE.base). The author's Verdin engine, shipped in the same repo, Apache-2.0. LICENCE FLAG (NC): CC-BY-NC-4.0 (non-commercial) weights; listing a measured score is evaluation use.",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "victorbennett/Verdin-4B@f35e9c184864a2fd6f286166783b0321d95f23b0",
   "noul_decisive": {
    "acc_among_decisive": 0.8647540983606558,
    "decisive": 244,
    "decisive_rate": 0.6541554959785523,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.57547250683973,
     "cost": 62.17924458049355,
     "intelligence": 35.50482714952485,
     "speed": 45.8869739530126
    },
    "board_release": "v1.6.43",
    "capability": 59.540149828182294,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      17.04093820678688,
      27.484134104815304
     ],
     "B": [
      15.411632616514813,
      25.68836113806341
     ],
     "C": [
      11.833984865824224,
      19.08620423945507
     ]
    },
    "jevbench_score": 21.77748190312391,
    "measured_in": "v1.6.27",
    "method": "v1.6",
    "open_board_rank": 84,
    "rank": 117,
    "ranks": {
     "A": 117,
     "B": 130,
     "C": 135,
     "capability": 136
    },
    "scores": {
     "A": 21.77748190312391,
     "B": 20.00039057273753,
     "C": 15.123251321613825
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 35.47302223105216,
     "p50_s_raw": 17.661511115526082,
     "p95_s_adjusted": 72.67824198754965,
     "p95_s_raw": 36.264120993774824
    },
    "views": {
     "Balanced 33:33:33": 19.292352873017215,
     "Emphasis on Accuracy 60:20:20": 17.352655644184253,
     "Emphasis on Cost 20:20:60": 21.62260137257583,
     "Emphasis on Speed 20:60:20": 19.370063474123214,
     "Intelligence only": 15.078600269727296
    }
   },
   "rank": 115,
   "ranked": true,
   "ranks": {
    "A": 115,
    "B": 116,
    "C": 121,
    "capability": 152
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/victorbennett/Verdin-4B",
   "scores": {
    "A": 29.859394724952654,
    "B": 26.30167843532924,
    "C": 20.735690781217123
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2660857129998476,
    "p50_s_raw": 0.0580428564999238,
    "p95_s_adjusted": 0.35803127409999425,
    "p95_s_raw": 0.10401563704999713
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 61.523990722530975,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 9.571972167592913
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 61.523990722530975
      },
      "calibration_gen": {
       "accuracy": 0.42574257425742573,
       "cc": 14.794406109571346,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": 4.0000000000000036,
        "score": 13.553950036031104
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.794406109571346
      },
      "composite_gen": {
       "accuracy": 0.504950495049505,
       "cc": 7.520948912108281,
       "cc_by_type": {
        "choice": 57.72357723577236,
        "noul": -48.14814814814815,
        "score": 12.987417648700639
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 7.520948912108281
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 73.41803255349417,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 60.25409766048252
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 73.41803255349417
      },
      "jaggedness": {
       "accuracy": 0.46111111111111114,
       "cc": 9.326554298072596,
       "cc_by_type": {
        "choice": 37.92107795957652,
        "noul": -40.54054054054054,
        "score": 30.59912547518181
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 9.326554298072596
      },
      "llm_router": {
       "accuracy": 0.7507987220447284,
       "cc": 59.92658024610777,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 58.762886597938135,
        "score": 51.922903273078646
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 59.92658024610777
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 0.44581408543078044,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25.0,
        "score": 0.5642463800036857
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.44581408543078044
      },
      "multilingual": {
       "accuracy": 0.6254545454545455,
       "cc": 34.88131457686409,
       "cc_by_type": {
        "choice": 58.9175089664167,
        "noul": 13.888888888888884,
        "score": 31.8375458752867
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.88131457686409
      },
      "negation_pair_gen": {
       "accuracy": 0.47674418604651164,
       "cc": 6.166223129941823,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -26.66666666666667,
        "score": -1.5013306101745227
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 6.166223129941823
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 66.91900260860074,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100.0,
        "score": 30.787037855832256
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 66.91900260860074
      },
      "usecase": {
       "accuracy": 0.509090909090909,
       "cc": 24.52631268483532,
       "cc_by_type": {
        "choice": 54.65686274509807,
        "noul": -8.333333333333337,
        "score": 27.25540864274124
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 24.52631268483532
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 5.110110788373182,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50.0,
        "score": -58.47919144440426
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 5.110110788373182
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.5384615384615384,
       "cc": 13.276437832483587,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 14.637301208664288
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 13.276437832483587
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 53.55726914887933,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 75.0,
        "score": 33.45941806610703
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 53.55726914887933
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.344756984735273,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 97.74004376612389
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.344756984735273
      },
      "en": {
       "accuracy": 0.5881841876629018,
       "cc": 31.086408945268783,
       "cc_by_type": {
        "choice": 56.50529039186729,
        "noul": 7.913669064748197,
        "score": 28.840267379190866
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.086408945268783
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 56.4252784262924,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": -11.148308277403407
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 56.4252784262924
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 5.744506156424248,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 48.927507540311
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 5.744506156424248
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 34.65178850066019,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0.0,
        "score": 67.48477726668645
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 34.65178850066019
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 34.34458127099561,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 36.55329688561254
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 34.34458127099561
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 68.88724101042284,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 20.691341113887418
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.88724101042284
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 51.95211351224185,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 80.0,
        "score": 33.04458517532763
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 51.95211351224185
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 5.151819689710763,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 15.455459069132294
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 5.151819689710763
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.68822847084569,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 78.0646854125371
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.68822847084569
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 23.297635944016566,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 55.17429311343498
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 23.297635944016566
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 40.04671144489828,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 19.999999999999996,
        "score": 22.99244072487682
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 40.04671144489828
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 51.04746948866693,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 35.570222255040264
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 51.04746948866693
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 19.073419744761807,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 53.81600391513648
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 19.073419744761807
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 25.79911330832609,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 20.87560079454349
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 25.79911330832609
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 66.65097394291317,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 32.21098634486855
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 66.65097394291317
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 14.386174718477827,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": 1.9155863023261444
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.386174718477827
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.668,
       "cc": 57.117434317533835,
       "cc_by_type": {
        "choice": 57.117434317533835
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 57.117434317533835
      },
      "noul": {
       "accuracy": 0.5626666666666666,
       "cc": 12.53333333333333,
       "cc_by_type": {
        "noul": 12.53333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 12.53333333333333
      },
      "score": {
       "accuracy": 0.5013333333333333,
       "cc": 29.091210398518484,
       "cc_by_type": {
        "score": 29.091210398518484
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 29.091210398518484
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.421017262210398,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": 0.7369482133688043
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 1.696566639321733,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -11.111111111111116,
        "score": -9.265027480240452
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.696566639321733
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 53.72603827573835,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 43.698440030467054
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 53.72603827573835
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -1.3767897429531726,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 22.770215566462127
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": -8.940606953354225,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100.0,
        "score": 31.14919363269094
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -7.0516993000313795,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -33.333333333333336,
        "score": 0.22854989864800057
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 26.75005211407454,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 23.947635333820262
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 26.75005211407454
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -8.642668549110079,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100.0,
        "score": 36.78976438751296
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 20.188784788594514,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 19.999999999999996,
        "score": 23.89968769911688
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 20.188784788594514
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 41.54483291107334,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 22.207660760343508
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 41.54483291107334
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 37.187525164753104,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": 1.4850561144143515
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 37.187525164753104
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -5.059825983930627,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100.0,
        "score": 17.07858656433716
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 27.97647267533547,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -19.999999999999996,
        "score": 26.962047584547655
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 27.97647267533547
      },
      "model_routing": {
       "accuracy": 0.7788018433179723,
       "cc": 61.48579662948972,
       "cc_by_type": {
        "choice": 81.2987012987013,
        "noul": 63.63636363636365,
        "score": 39.52232495340419
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 61.48579662948972
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 5.86650020310321,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 52.14844494712153
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 5.86650020310321
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 62.59236833041763,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100.0,
        "score": 21.203288835264043
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.59236833041763
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": 1.030281616673367,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": 12.204145342630934
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 1.030281616673367
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 15.64194265939215,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": 0.0,
        "score": 19.389596094118478
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 15.64194265939215
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 73.80007572716669,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 71.40022718150007
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 73.80007572716669
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.79608109938583,
      84.57493113282798
     ],
     "I": [
      32.107645595824906,
      39.19687931973971
     ],
     "capability": [
      55.812920523721445,
      60.99580491591795
     ],
     "composite": {
      "A": [
       23.29437819037973,
       37.91142806589149
      ],
      "B": [
       20.216299228222162,
       34.03666918233901
      ],
      "C": [
       16.17665152109704,
       26.32738060131354
      ]
     },
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.75263461013449,
     "capability": 59.59877475135272,
     "intelligence": 35.44491489257096
    },
    "per_type": {
     "choice": {
      "accuracy": 0.668,
      "calibration": 84.42288010235657,
      "cc": 57.117434317533835,
      "cc_by_type": {
       "choice": 57.117434317533835
      },
      "cc_tiered": 61.107358564628726,
      "cc_tiered_open": 69.12490790461862,
      "cc_tiered_sealed": 53.08980922463883,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.117434317533835
     },
     "noul": {
      "accuracy": 0.5626666666666666,
      "calibration": 78.62185714020772,
      "cc": 12.53333333333333,
      "cc_by_type": {
       "noul": 12.53333333333333
      },
      "cc_tiered": 19.372870342628005,
      "cc_tiered_open": 37.862663906142174,
      "cc_tiered_sealed": 0.8830767791138365,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 12.53333333333333
     },
     "score": {
      "accuracy": 0.5013333333333333,
      "calibration": 88.21316658783921,
      "cc": 29.091210398518484,
      "cc_by_type": {
       "score": 29.091210398518484
      },
      "cc_tiered": 31.79392127825626,
      "cc_tiered_open": 39.981006090427115,
      "cc_tiered_sealed": 23.606836466085404,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.091210398518484
     }
    },
    "raw_on_A": {
     "calibration": 74.90958669726717,
     "capability": 56.9855068754991,
     "intelligence": 39.061427053731016
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.222764206614034,
    "Emphasis on Accuracy 60:20:20": 22.473588448079607,
    "Emphasis on Cost 20:20:60": 28.701438210307703,
    "Emphasis on Speed 20:60:20": 32.4004685205904,
    "Intelligence only": 17.812373989297615
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 91.37586990638505,
    "cost": 29.839473214267542,
    "intelligence": 72.92342726693953,
    "speed": 87.00758628687251
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03405766539472167,
      "ece_score": 93.18846692105566,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 91.94595402877492,
      "score_before_time_limit": 93.18846692105566,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.1466666666666666,
      "brier": 0.06360963348259217,
      "ece": 0.056426602196387884,
      "n": 375,
      "score": 88.24153460306523,
      "score_before_time_limit": 88.71467956072242,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.24200000000000008,
      "n": 375,
      "nrps": 0.04397667161509103,
      "score": 93.94012108731499,
      "score_before_time_limit": 94.69770270898688,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.031034637102585673
     }
    },
    "score": 91.37586990638505
   },
   "capability": 82.1496485866623,
   "composite_ci95": {
    "A": [
     20.08193039290129,
     20.62307569100094
    ],
    "B": [
     20.821662668052426,
     21.677708470739717
    ],
    "C": [
     20.08193039290129,
     20.62307569100094
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff for these weights): the frozen Qwen/Qwen3.8-27B market reference of the published v1.6 field, USD 0.42 per 1M input and USD 3.00 per 1M output - the same pair the published pplx-decider-v1-1-27b and torchcast-decision-27b rows carry - x this row's own measured tokens on the common 1,485-item basis (1,951,205 input tokens; the author's server reports 0 output tokens because nothing is generated). USD 0.218114 per 1,000 decisions on the common basis (USD 0.5463374 over all 1,500 items), about 3.4x the published v1.6 ceiling of USD 0.0646 per 1,000 decisions, so this row is outside the default cost-capped Capability ranking and the squared cost gate keeps its composite small. Estimated, not charged; not a quote from the author or any host. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.218114 per 1,000 decisions (3.377x the official cost cap USD 0.064595), against USD 0.546337 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5463374
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.21811434343434347
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "vLLM-SR d3",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (confidential-computing mode, attested), NOT the 1x H100 80 GB on which this draw's other GPU rows and the published Perplexity Decider and Torchcast-Decision-27B rows were timed - no single H100 was available at measurement time - so this row's Speed axis rests on different hardware from those rows (some other published v1.6 rows, for example xor-26b-a4b and rene-1-31b-fp8, were also timed on an RTX PRO 6000). The author's own unmodified d3_server.py (plain CUDA / PyTorch path; the author's AMD ROCm fast path does not apply on CUDA) in an offline container with --network none, Hub offline mode and all credentials unset, weights verified against the author's MODEL_MANIFEST sha256 list; driven by the unchanged pinned pod_driver_v16 with one official warm-up and a sequential scoring loop. The standard self-hosted x2 + 0.15 s own-server adjustment (an assumption, not measured) is applied on top of the raw p50/p95. Measured 12 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b); VM destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.42164858078797,
    "I_sealed": 67.42520595309108,
    "base": 72.92342726693953,
    "excess": 1.1569158573411826,
    "gap": 10.996442627696894,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 88.14184875359072,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 86.78829820698331,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 77.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 70.04580230278285,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.43476019846001,
       "hard": 76.385216737932,
       "judge": 75.95310460653639,
       "standard": 50.31154102901563
      }
     },
     "sealed|choice": {
      "cc": 80.69575983859798,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 71.89776801202729,
       "judge": 83.2517140058766,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 56.25893200983268,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 43.636363636363626,
       "judge": 53.846153846153854,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 65.32092601084263,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.15753121858664,
       "hard": 62.596109395819276,
       "judge": 61.28832264025312,
       "standard": 67.40116169290158
      }
     }
    }
   },
   "jevbench_score": 20.450399737040453,
   "key": "vllm-sr-d3",
   "last_measured_on": "2026-10-12",
   "licence": "Apache-2.0 (weights, model card and NOTICE; base Qwen/Qwen3.8-27B Apache-2.0). Training data not disclosed",
   "listing": "ranked",
   "measured_in": "v1.6.16",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/d3@5ce995dfb7ff07da00fec4211eb7f0560f52eaf8",
   "noul_decisive": {
    "acc_among_decisive": 0.9623824451410659,
    "decisive": 319,
    "decisive_rate": 0.8552278820375335,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 92.20028306358832,
     "cost": 29.839473214267542,
     "intelligence": 73.73157285955872,
     "speed": 87.36893361347018
    },
    "board_release": "v1.6.43",
    "capability": 82.96592796157353,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.171426588851887,
      20.699584134625216
     ],
     "B": [
      20.946864446552823,
      21.780666403522886
     ],
     "C": [
      20.171426588851887,
      20.699584134625216
     ]
    },
    "jevbench_score": 20.53757421097071,
    "measured_in": "v1.6.16",
    "method": "v1.6",
    "open_board_rank": 94,
    "rank": 128,
    "ranks": {
     "A": 128,
     "B": 119,
     "C": 110,
     "capability": 11
    },
    "scores": {
     "A": 20.53757421097071,
     "B": 21.473456539895587,
     "C": 20.53757421097071
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.36749324100005654,
     "p50_s_raw": 0.10874662050002826,
     "p95_s_adjusted": 0.4987205717000961,
     "p95_s_raw": 0.17436028585004806
    },
    "views": {
     "Balanced 33:33:33": 18.257915673012125,
     "Emphasis on Accuracy 60:20:20": 20.792305962031904,
     "Emphasis on Cost 20:20:60": 14.18430475741736,
     "Emphasis on Speed 20:60:20": 21.87364349863337,
     "Intelligence only": 26.26006480299088
    }
   },
   "rank": 145,
   "ranked": true,
   "ranks": {
    "A": 145,
    "B": 139,
    "C": 128,
    "capability": 17
   },
   "repo": "https://huggingface.co/vllm-sr/d3",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.450399737040453,
    "B": 21.358590030107145,
    "C": 20.450399737040453
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3702694839998003,
    "p50_s_raw": 0.11013474199990014,
    "p95_s_adjusted": 0.5379271327999277,
    "p95_s_raw": 0.19396356639996384
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 79.70393292683865,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 47.44513211384924
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 79.70393292683865
      },
      "calibration_gen": {
       "accuracy": 0.7623762376237624,
       "cc": 61.321415531109665,
       "cc_by_type": {
        "choice": 58.642629904559925,
        "noul": 52.0,
        "score": 73.32161668876907
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.321415531109665
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 88.97552391667175,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 80.83804419807305
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 88.97552391667175
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.84803221342577,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 97.54409664027729
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.84803221342577
      },
      "jaggedness": {
       "accuracy": 0.7055555555555556,
       "cc": 54.01362633380514,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": 29.729729729729737,
        "score": 68.4035458068156
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 54.01362633380514
      },
      "llm_router": {
       "accuracy": 0.939297124600639,
       "cc": 85.14867822344807,
       "cc_by_type": {
        "choice": 96.03768600862904,
        "noul": 100.0,
        "score": 59.40834866171518
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 85.14867822344807
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 40.30049701458755,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 0.0,
        "score": 58.01489310561831
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 40.30049701458755
      },
      "multilingual": {
       "accuracy": 0.7890909090909091,
       "cc": 63.79693472202507,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 44.44444444444444,
        "score": 65.53129614158513
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.79693472202507
      },
      "negation_pair_gen": {
       "accuracy": 0.9069767441860465,
       "cc": 78.16865484806344,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 73.33333333333334,
        "score": 61.172631210856984
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.16865484806344
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 86.04495127304953,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 70.14686583116061
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.04495127304953
      },
      "usecase": {
       "accuracy": 0.7727272727272727,
       "cc": 61.382363634162466,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 37.5,
        "score": 68.70591443189917
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 61.382363634162466
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 69.16376241279302,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 26.53890628599814
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 69.16376241279302
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 46.72533851096023,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 6.842682199547367
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 46.72533851096023
      },
      "de": {
       "accuracy": 0.875,
       "cc": 85.59895551835274,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 80.69067186479275
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.59895551835274
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.49221077651324,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 99.9961854021654
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.49221077651324
      },
      "en": {
       "accuracy": 0.8427454387489139,
       "cc": 73.66051265515044,
       "cc_by_type": {
        "choice": 82.86572045740228,
        "noul": 68.34532374100719,
        "score": 69.77049376704187
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 73.66051265515044
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.27374935392363,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 29.60917628363061
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.27374935392363
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.71287671897267,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.80529682358468
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.71287671897267
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 83.92352369573172,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 94.12351226366575
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 83.92352369573172
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 50.875059370970995,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25.0,
        "score": 44.384954649225826
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 50.875059370970995
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 73.4076618326374,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 34.25260358053105
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.4076618326374
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 38.966499106440274,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 16.899497319320812
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.966499106440274
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 19.173970994902465,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 57.52191298470741
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 19.173970994902465
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.92564862400029,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.77694587200085
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.92564862400029
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -20.032834215525288,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": -34.124476672549896
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 79.398514682308,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80.0,
        "score": 65.81297951031802
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 79.398514682308
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 79.23908352196145,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 69.53811090766393
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 79.23908352196145
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 47.20572405326843,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 47.149087053422335
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 47.20572405326843
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 85.40756411534664,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 56.22269234603995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 85.40756411534664
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.18093252705013,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 99.05892661340849
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.18093252705013
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 48.40707026933146,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 11.887877474661067
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.40707026933146
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8653333333333333,
       "cc": 82.60586693201171,
       "cc_by_type": {
        "choice": 82.60586693201171
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.60586693201171
      },
      "noul": {
       "accuracy": 0.8186666666666667,
       "cc": 63.733333333333334,
       "cc_by_type": {
        "noul": 63.733333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 63.733333333333334
      },
      "score": {
       "accuracy": 0.7706666666666667,
       "cc": 67.41226190218106,
       "cc_by_type": {
        "score": 67.41226190218106
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.41226190218106
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 19.11264958464173,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -50.0,
        "score": 23.337948753925186
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 19.11264958464173
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 89.26555045427034,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 67.79665136281103
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 89.26555045427034
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 34.48589320988469,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 2.2381674345321345
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 34.48589320988469
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -1.3162243525891217,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -67.10656779460946
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 39.51821695055512,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 80.8734914313755
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 39.51821695055512
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 40.793860182167386,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 1.6268635653700847
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 40.793860182167386
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 53.855675565990985,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 66.88915554951359
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 53.855675565990985
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 70.98778127144608,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50.0,
        "score": 73.4163054868121
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 70.98778127144608
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 14.027076831306962,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 64.58123049392088
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 14.027076831306962
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 79.02166050259233,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 70.39831484111032
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 79.02166050259233
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 70.03571035914229,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 43.957260276393285
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 70.03571035914229
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 17.71047097327813,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 48.830337651017196
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 17.71047097327813
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 61.7232011127565,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 65.16960333826947
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 61.7232011127565
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 79.70594033783782,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 46.82344871914113
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.70594033783782
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 13.96550875487595,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 76.44547060243976
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 13.96550875487595
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 59.29379240056826,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 77.97422771238257
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 59.29379240056826
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 37.6695454341995,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 27.7869614257512
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 37.6695454341995
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 28.5849244629175,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 77.05912121483945
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 28.5849244629175
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.92396962090108,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.77190886270323
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.92396962090108
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.99802173783495,
      91.86975408101644
     ],
     "I": [
      68.29342272755383,
      76.59687229252359
     ],
     "capability": [
      78.2543213807504,
      83.72614571299034
     ],
     "composite": {
      "A": [
       20.08193039290129,
       20.62307569100094
      ],
      "B": [
       20.821662668052426,
       21.677708470739717
      ],
      "C": [
       20.08193039290129,
       20.62307569100094
      ]
     },
     "seed": 83
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.37586990638505,
     "capability": 82.1496485866623,
     "intelligence": 72.92342726693953
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8653333333333333,
      "calibration": 91.94595402877492,
      "cc": 82.60586693201171,
      "cc_by_type": {
       "choice": 82.60586693201171
      },
      "cc_tiered": 84.41880429609435,
      "cc_tiered_open": 88.14184875359072,
      "cc_tiered_sealed": 80.69575983859798,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.60586693201171
     },
     "noul": {
      "accuracy": 0.8186666666666667,
      "calibration": 88.24153460306523,
      "cc": 63.733333333333334,
      "cc_by_type": {
       "noul": 63.733333333333334
      },
      "cc_tiered": 66.66811334791151,
      "cc_tiered_open": 77.07729468599035,
      "cc_tiered_sealed": 56.25893200983268,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 63.733333333333334
     },
     "score": {
      "accuracy": 0.7706666666666667,
      "calibration": 93.94012108731499,
      "cc": 67.41226190218106,
      "cc_by_type": {
       "score": 67.41226190218106
      },
      "cc_tiered": 67.68336415681273,
      "cc_tiered_open": 70.04580230278285,
      "cc_tiered_sealed": 65.32092601084263,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.41226190218106
     }
    },
    "raw_on_A": {
     "calibration": 89.95256547604913,
     "capability": 80.86122053647321,
     "intelligence": 71.7698755968973
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.19640036187322,
    "Emphasis on Accuracy 60:20:20": 20.67199915561807,
    "Emphasis on Cost 20:20:60": 14.161988052441243,
    "Emphasis on Speed 20:60:20": 21.795228194698396,
    "Intelligence only": 25.972237556000575
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 90.35330442455489,
    "cost": 29.839473214267542,
    "intelligence": 74.82897422466625,
    "speed": 85.64377521409548
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05697692176294583,
      "ece_score": 88.60461564741084,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.60461564741084,
      "score_before_time_limit": 88.60461564741084,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.13404825737265413,
      "brier": 0.0619027410142049,
      "ece": 0.061196749215429905,
      "n": 373,
      "score": 87.76065015691402,
      "score_before_time_limit": 87.76065015691402,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.946236559139785,
      "interval90_width": 0.1843637992831541,
      "n": 372,
      "nrps": 0.04245289733343758,
      "score": 94.6946474693398,
      "score_before_time_limit": 94.6946474693398,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03182707663988325
     }
    },
    "score": 90.35330442455489
   },
   "capability": 82.59113932461057,
   "composite_ci95": {
    "A": [
     20.112736271788723,
     20.64116616700081
    ],
    "B": [
     20.943018265596475,
     21.761090092757485
    ],
    "C": [
     20.112736271788723,
     20.64116616700081
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff for these weights): declared base Qwen/Qwen3.8-27B -> qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output - the same pair the published vllm-sr-d3 row on this draw carries - x this row's own measured tokens on the common 1,485-item basis (771,190 input tokens on the 1,485 answered items; the 15 items over the author's 32,768-token default cap are refused by the author's own server with 0 tokens counted; output 0 because nothing is generated). Estimated, not charged; not a quote from the author or any host. OVER THE OFFICIAL COST CAP: USD 0.21811434343434347 per 1,000 decisions on the 1,485 common items, 3.38x the official cap USD 0.06459465517241379; ranked and disclosed following the published vllm-sr-d3 precedent. The frozen Qwen3.8-27B pair is also carried by published startlux-decision-v1.1-27b-instant and startlux-decision-v1.1-27b-autothink. ALL-ITEMS DIAGNOSTIC: USD 0.2159332 per 1,000 decisions over the full 1,500-item denominator, 3.34x the official cap; the published cost above uses the same remaining 1,485 items as every other row. All items still count for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.21593320000000002
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.21811434343434347
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "vLLM-SR d3-turbo",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-19",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN, pay-as-you-go non-spot), the same GPU class the published vllm-sr-d3 row was timed on (that row ran in confidential-computing mode, this one did not; raw p50 0.144 s vs d3's 0.109 s). The author's own unmodified stdlib d3_server.py (plain CUDA / PyTorch path; the author's ROCm fast path does not apply on CUDA) in an offline container with --network none (cut proven from inside the measuring container before the sealed input: loopback only, no default route, TCP and DNS fail), Hub offline mode and all credentials unset, every weight shard, readout, tokenizer, config and code file sha256-verified against the author's MODEL_MANIFEST.json; driven by the unchanged pinned pod_driver_v16. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 19 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.94726417219914,
    "I_sealed": 69.71068427713337,
    "base": 74.82897422466625,
    "excess": 0.3970531247100624,
    "gap": 10.236579895065773,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 87.81849238331058,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 92.089650626236,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 80.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 91.30434782608697,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 71.9460054472965,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 74.70222801620412,
       "hard": 77.18034928834129,
       "judge": 73.57972551845668,
       "standard": 57.6486263740128
      }
     },
     "sealed|choice": {
      "cc": 81.76677045405546,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 74.49982652943217,
       "judge": 85.89618021547503,
       "standard": 87.66083104830204
      }
     },
     "sealed|noul": {
      "cc": 58.81486706808374,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 47.272727272727266,
       "judge": 53.846153846153854,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 68.55041530926088,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.70603996671883,
       "hard": 64.62159006380274,
       "judge": 68.07303559090225,
       "standard": 69.04632304898617
      }
     }
    }
   },
   "jevbench_score": 20.462833858610495,
   "key": "vllm-sr-d3-turbo",
   "last_measured_on": "2026-10-19",
   "licence": "Apache-2.0 (weights, model card and NOTICE; base Qwen/Qwen3.8-27B family Apache-2.0). Training data not disclosed",
   "listing": "ranked",
   "measured_in": "v1.6.31",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/d3-turbo@689b3af198baaf6f11110be38a9daa7188dfd8ad",
   "noul_decisive": {
    "acc_among_decisive": 0.9628482972136223,
    "decisive": 323,
    "decisive_rate": 0.8659517426273459,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.35330442455489,
     "cost": 29.839473214267542,
     "intelligence": 74.82897422466625,
     "speed": 86.03728492328564
    },
    "board_release": "v1.6.43",
    "capability": 82.59113932461057,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.127911685633038,
      20.65714979248832
     ],
     "B": [
      20.956179967315336,
      21.775300465066284
     ],
     "C": [
      20.127911685633038,
      20.65714979248832
     ]
    },
    "jevbench_score": 20.47854238643572,
    "measured_in": "v1.6.31",
    "method": "v1.6",
    "open_board_rank": 95,
    "rank": 129,
    "ranks": {
     "A": 129,
     "B": 120,
     "C": 111,
     "capability": 13
    },
    "scores": {
     "A": 20.47854238643572,
     "B": 21.473183288464952,
     "C": 20.47854238643572
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4387955279999233,
     "p50_s_raw": 0.14439776399996163,
     "p95_s_adjusted": 0.5675567784000123,
     "p95_s_raw": 0.20877838920000613
    },
    "views": {
     "Balanced 33:33:33": 18.264704402771564,
     "Emphasis on Accuracy 60:20:20": 20.894663067289894,
     "Emphasis on Cost 20:20:60": 14.186762674816508,
     "Emphasis on Speed 20:60:20": 21.784657900798422,
     "Intelligence only": 26.65091271040641
    }
   },
   "rank": 144,
   "ranked": true,
   "ranks": {
    "A": 144,
    "B": 137,
    "C": 127,
    "capability": 15
   },
   "repo": "https://huggingface.co/vllm-sr/d3-turbo",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.462833858610495,
    "B": 21.45936437369482,
    "C": 20.462833858610495
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.45193122999992286,
    "p50_s_raw": 0.15096561499996142,
    "p95_s_adjusted": 0.6033233377001693,
    "p95_s_raw": 0.22666166885008465
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 94.95104382165277,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 93.18646479829165
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 94.95104382165277
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 60.4811415272761,
       "cc_by_type": {
        "choice": 68.18663838812301,
        "noul": 43.99999999999999,
        "score": 69.2567861937053
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 60.4811415272761
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 90.19347841839642,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 84.4919077032471
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.19347841839642
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 99.81665420977471,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.44996262932413
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.81665420977471
      },
      "jaggedness": {
       "accuracy": 0.7277777777777777,
       "cc": 56.78661196056641,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 35.13513513513513,
        "score": 72.76079314309922
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 56.78661196056641
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 84.48235737075346,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 95.8762886597938,
        "score": 60.74063464556334
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.48235737075346
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 44.32687494250607,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": -25.0,
        "score": 82.72289286875534
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 44.32687494250607
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 67.1694662456961,
       "cc_by_type": {
        "choice": 83.37137267688294,
        "noul": 50.0,
        "score": 68.13702606020536
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 67.1694662456961
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 75.39042370786824,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 73.33333333333334,
        "score": 52.83793779027138
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 75.39042370786824
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 88.56210813454986,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100.0,
        "score": 71.69233040965554
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 88.56210813454986
      },
      "usecase": {
       "accuracy": 0.7818181818181819,
       "cc": 64.20114507147237,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 45.83333333333333,
        "score": 72.5053959987308
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.20114507147237
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 77.53113811828422,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100.0,
        "score": 70.68865245009076
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 77.53113811828422
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 50.26735599928611,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 17.468734664524987
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 50.26735599928611
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 83.29394929791897,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100.0,
        "score": 81.74025497340291
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.29394929791897
      },
      "el": {
       "accuracy": 0.8,
       "cc": 72.18845079261344,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 33.33333333333333,
        "score": 99.99179558081985
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 72.18845079261344
      },
      "en": {
       "accuracy": 0.8496959165942659,
       "cc": 75.0660808413856,
       "cc_by_type": {
        "choice": 82.86572045740228,
        "noul": 70.50359712230217,
        "score": 71.82892494445237
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.0660808413856
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.79728599021023,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 40.96771441435008
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.79728599021023
      },
      "fi": {
       "accuracy": 1.0,
       "cc": 99.97222202004065,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.91666606012194
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 99.97222202004065
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 91.69144203830236,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 96.25079670314238
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 91.69144203830236
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 56.7440794132958,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 0.0,
        "score": 70.23223823988742
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 56.7440794132958
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 78.3981666466817,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 35.19449994004508
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 78.3981666466817
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 36.9676471604448,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 19.999999999999996,
        "score": 5.200002640984913
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.9676471604448
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 17.49673391023856,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 52.49020173071568
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 17.49673391023856
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.95856075147947,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.87568225443837
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.95856075147947
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -15.46724452134869,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -46.40173356404607
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8717948717948718,
       "cc": 75.54086356569267,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 74.24002616047201
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 75.54086356569267
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 79.12997344945892,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 69.21078069015635
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 79.12997344945892
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 61.17170090469063,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 49.0470176076889
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 61.17170090469063
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 66.2952315244401,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 48.88569457332029
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 66.2952315244401
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 88.06941269523666,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 96.46630260183903
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 88.06941269523666
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 47.48723060422734,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 9.128358479348709
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 47.48723060422734
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8693333333333333,
       "cc": 83.12252434987275,
       "cc_by_type": {
        "choice": 83.12252434987275
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.12252434987275
      },
      "noul": {
       "accuracy": 0.8293333333333334,
       "cc": 65.86666666666667,
       "cc_by_type": {
        "noul": 65.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 65.86666666666667
      },
      "score": {
       "accuracy": 0.8026666666666666,
       "cc": 69.99615063036917,
       "cc_by_type": {
        "score": 69.99615063036917
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 69.99615063036917
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 22.494329990087607,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -25.0,
        "score": 24.482989970262825
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 22.494329990087607
      },
      "code_linting": {
       "accuracy": 1.0,
       "cc": 96.10834802982788,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 88.32504408948365
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 96.10834802982788
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 26.49115104816521,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50.0,
        "score": -5.485896449000305
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 26.49115104816521
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -2.395818046525098,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -70.34534887641739
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.98936592070195,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 85.286938341816
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.98936592070195
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 55.92594796066326,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 47.023126900857726
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 55.92594796066326
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 62.11035211650731,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 63.08175662963396
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 62.11035211650731
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 73.43481413884172,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50.0,
        "score": 80.75740408899902
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 73.43481413884172
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 47.33761496863716,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -19.999999999999996,
        "score": 82.84617823924482
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 47.33761496863716
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 75.98050714558066,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 61.27485477007531
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 75.98050714558066
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.25946054476482,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 57.8507330554831
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.25946054476482
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 13.00854804556269,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 34.72456886787087
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 13.00854804556269
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 61.743150197630165,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 76.74576537216112
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 61.743150197630165
      },
      "model_routing": {
       "accuracy": 0.8986175115207373,
       "cc": 80.12327163220625,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 48.07544260224647
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.12327163220625
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 52.81259558276948,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 81.47041630684969
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 52.81259558276948
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 65.81280362944747,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 80.81816947701466
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 65.81280362944747
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 24.298872150201333,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50.0,
        "score": 37.674941573756705
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 24.298872150201333
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 24.248186712768014,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 78.5416615875794
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 24.248186712768014
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.94054161265387,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.8216248379616
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.94054161265387
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.55277391297311,
      91.6227365906473
     ],
     "I": [
      70.21559237844008,
      78.18480469530246
     ],
     "capability": [
      78.79489368820927,
      84.4933851527466
     ],
     "composite": {
      "A": [
       20.112736271788723,
       20.64116616700081
      ],
      "B": [
       20.943018265596475,
       21.761090092757485
      ],
      "C": [
       20.112736271788723,
       20.64116616700081
      ]
     },
     "seed": 84
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.35330442455489,
     "capability": 82.59113932461057,
     "intelligence": 74.82897422466625
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8693333333333333,
      "calibration": 88.60461564741084,
      "cc": 83.12252434987275,
      "cc_by_type": {
       "choice": 83.12252434987275
      },
      "cc_tiered": 84.79263141868302,
      "cc_tiered_open": 87.81849238331058,
      "cc_tiered_sealed": 81.76677045405546,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.12252434987275
     },
     "noul": {
      "accuracy": 0.8293333333333334,
      "calibration": 87.76065015691402,
      "cc": 65.86666666666667,
      "cc_by_type": {
       "noul": 65.86666666666667
      },
      "cc_tiered": 69.44608087703705,
      "cc_tiered_open": 80.07729468599035,
      "cc_tiered_sealed": 58.81486706808374,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 65.86666666666667
     },
     "score": {
      "accuracy": 0.8026666666666666,
      "calibration": 94.6946474693398,
      "cc": 69.99615063036917,
      "cc_by_type": {
       "score": 69.99615063036917
      },
      "cc_tiered": 70.24821037827869,
      "cc_tiered_open": 71.9460054472965,
      "cc_tiered_sealed": 68.55041530926088,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 69.99615063036917
     }
    },
    "raw_on_A": {
     "calibration": 89.05513895349148,
     "capability": 81.74574955505761,
     "intelligence": 74.43636015662376
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.248045806111314,
    "Emphasis on Accuracy 60:20:20": 20.88157850042968,
    "Emphasis on Cost 20:20:60": 14.180729546933513,
    "Emphasis on Speed 20:60:20": 21.7420457889593,
    "Intelligence only": 26.65091271040641
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "andyzhang232 (Hugging Face)",
   "axes": {
    "calibration": 91.33817262545804,
    "cost": 40.983187840250785,
    "intelligence": 71.46073788705554,
    "speed": 82.45454940142191
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.046153910328050815,
      "ece_score": 90.76921793438983,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.76921793438983,
      "score_before_time_limit": 90.76921793438983,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.13404825737265413,
      "brier": 0.04568060550795997,
      "ece": 0.03819846894711239,
      "n": 373,
      "score": 92.36030621057752,
      "score_before_time_limit": 92.36030621057752,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.23812724014336917,
      "n": 372,
      "nrps": 0.050897974834229455,
      "score": 90.88499373140681,
      "score_before_time_limit": 90.88499373140681,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06570107526881724
     }
    },
    "score": 91.33817262545804
   },
   "capability": 81.3994552562568,
   "composite_ci95": {
    "A": [
     42.76962119886615,
     44.17072712154098
    ],
    "B": [
     43.12257834116863,
     45.31558883084057
    ],
    "C": [
     42.76962119886615,
     44.17072712154098
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the rates for this base string are identical in the earlier module 81e3578f): base string google/gemma-4-31B-it (the card's base_model) priced as google/gemma-4-31b-it USD 0.09/M input, USD 0.34/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of our shim, which mirrors the author's server (input_tokens = length of the rendered prompt token ids incl. system message, options, repeated user text and chat template; output_tokens = 0, as the author's server reports with thinking off, although vLLM samples and discards one token per request: 1,485 completion tokens in total, kept in our per-row audit log, not billed). Per row, shim usage = vLLM prompt_tokens = our offline count through the same shim and the pinned tokenizer: 1,530,069 input tokens over the 1,485 answered items, and 15 refusals at the pre-registered ids and positions (HTTP 422, 0 tokens). OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.092731 per 1,000 decisions on the common basis (USD 0.091804 over all 1,500 items) is 1.436x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them); their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.09180413999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.09273145454545455
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "AJev lm6q (Gemma 4 31B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. Stock vLLM 0.30.0 and our shim (mounted read-only, sha re-checked on the node) in ONE docker container with network none (the cut proven from inside at start and again by docker exec from outside: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, only the output directory writable, env -i with every credential-named variable empty, HF offline flags, the sealed input not present until the smoke and the time gate had passed (input sha256 dfd36720... checked on the node and inside the container); one request at a time, timed over loopback on the VM. Measured 22 Oct 2026 (VM reserved 09:44Z, scored run 09:55-10:01Z, released 10:01Z, about USD 0.76) on the frozen v1.6-regular-20261012 self-hosted input; node shredded and VM removed afterwards (teardown confirmed).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.10537789504917,
    "I_sealed": 69.81609787906191,
    "base": 71.46073788705554,
    "excess": -6.550246754368452,
    "gap": 3.2892800159872593,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 78.86591280029805,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 75.46398238439761,
       "judge": 72.31377719182598,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 75.82125603864733,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 64.62896484620214,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.51666666666668,
       "hard": 76.90657407407407,
       "judge": 58.80593220338983,
       "standard": 43.36444444444443
      }
     },
     "sealed|choice": {
      "cc": 84.1552380959759,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 81.78559037816582,
       "judge": 84.1332027424094,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 64.82841399465178,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 49.09090909090909,
       "judge": 72.30769230769232,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 60.46464154655807,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.05447368421052,
       "hard": 61.654957746478914,
       "judge": 55.02755208333334,
       "standard": 56.944727272727256
      }
     }
    }
   },
   "jevbench_score": 43.718682863126155,
   "key": "ajev-gemma4-31b-lm6q",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card metadata; no LICENSE file in the repository; base google/gemma-4-31B-it: apache-2.0 in its card metadata with a link to the Gemma 4 licence), public, not gated; the card warns that some training data carries non-commercial or attribution terms",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "andyzhang232/ajev-gemma4-31b-lm6q@bdcfb05db02f48729f5f5a0b6b1c8cf6d6406d06",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9814241486068112,
    "decisive": 323,
    "decisive_rate": 0.8659517426273459,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 82,
   "ranked": true,
   "ranks": {
    "A": 82,
    "B": 78,
    "C": 69,
    "capability": 25
   },
   "repo": "https://huggingface.co/andyzhang232/ajev-gemma4-31b-lm6q",
   "scores": {
    "A": 43.718682863126155,
    "B": 44.51459519230017,
    "C": 43.718682863126155
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5768386019999753,
    "p50_s_raw": 0.21341930099998763,
    "p95_s_adjusted": 0.9851236474000756,
    "p95_s_raw": 0.4175618237000378
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "NO AUTHOR CODE WAS EXECUTED OR IMPORTED. The Hub repository holds a rank-64 LoRA adapter (safetensors + configs only, no .py, no auto_map): andyzhang232/ajev-gemma4-31b-lm6q at bdcfb05db02f48729f5f5a0b6b1c8cf6d6406d06 (Apache-2.0 per card, not merged), applied directly by stock vLLM 0.30.0 (image vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90, no derived image, no overlay wheels) on google/gemma-4-31B-it at 842da3794eaa0b77d5f08bae87a17459d91ff475. The request/response layer is OUR first-party shim ajev_systemone.py (sha256 e849fcdbaf0740f3a22b61108053572b3a76d3b35030aa39ed6b00f560bcbb9a), a rule-by-rule port of the author's published inference package (cmzy/ajev-infer at e2e723aba95a1f15d51966f43b53430372f49a4d, read as text only; the unit tests compare the author's constants by ast, never exec): request normalisation, prompt v3 (v2 layout with system message for choice questions of up to 26 options, v1 layout otherwise, user text repeated once when it is at most 12,000 characters), Gemma-4 chat template with thinking off, no truncation, lettered options A-Z then tokenizer-verified two-letter codes up to 255, readout over both single-token forms of each label combined by logsumexp, softmax over labels at T=1 (the adapter stores no temperatures). One /v1/completions call per decision (max_tokens 1, temperature 0, allowed_token_ids = all label forms, logprobs = their count). Server flags: --enable-lora --max-lora-rank 64 --max-loras 1 --lora-modules ajev-lm6q=<adapter snapshot> --dtype bfloat16 --max-model-len 40960 --gpu-memory-utilization 0.88 --enable-prefix-caching --max-logprobs 512 --logprobs-mode processed_logprobs --generation-config vllm --max-num-seqs 16 --max-num-batched-tokens 8192. Review: independent reviews of the shim and the runner preceded the run; their findings were fixed and re-reviewed, and the shim hash above is the one that ran. Weights: both snapshots downloaded revision-pinned and every file sha256/inventory-verified against MODEL_MANIFEST.json (sha256 dade7dfa4333fd15ef79c5a1682e4c3a3a6fd79a45035b251d6704a0492ad086): base 12 files, 62,578,686,256 bytes; adapter 5 files, 1,959,002,147 bytes; verdict OK for both. Smoke before any item: a separate smoke-only Verda pod (passed on attempt 12) and an in-container synthetic probe on the scored VM; no test item was ever used in a smoke. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry). SERVED BY OUR SHIM, NOT BY THE AUTHOR'S SERVER: the row measures the author's documented vLLM direct-LoRA serving path as re-implemented by us (stock vLLM 0.30.0 + our shim), not necessarily the configuration behind the author's self-reported 65.08 (the card says only that it was self-run with the official kit); there is no numeric parity check against the author's code (running it is not allowed here), only a rule-by-rule port plus tests. DISCLOSED DEVIATIONS from the author's server: D1 score probabilities are a dict keyed by level (the adapter needs a dict; the author returns a list), rounded to 4 decimals like the author; D2 a missing label token is an error instead of the author's -1e4 fill (cannot change an answer here, all label ids are requested); D3 requests go through vLLM's HTTP /v1/completions route instead of in-process generation; D4 an upstream 'maximum context length' 400 would be mapped to 422 (audited separately; never happened); D5 batch parameters --max-num-batched-tokens 8192 --max-num-seqs 16 are our pins, because the author's engine most likely runs 2,496 / 128 (source-derived, not verified live) and an explicit 2,048 is rejected by vLLM 0.30.0 for Gemma-4; this changes chunk boundaries and batch shape only; the context limit 40,960 follows the author's inference README for this 31B submission (the author's server default is 131,072); output_tokens is reported as 0 like the author. OPEN PARITY RISK (not tested): the adapter has v_proj LoRA on 50 of 60 layers and none on the 10 global layers where K and V share one projection; vLLM's fused qkv_proj loads K into both slots, so a K delta may not reach V as it would under Hugging Face PEFT. The smoke proved the adapter is really applied (LoRA-vs-base total variation up to 0.95 on a choice case) and that our readout equals an independent recomputation from raw upstream logprobs; during the smoke attempts the probe-only tolerances were loosened by us (a looser bound for cases with more than 26 options, 1e-6 to 1e-3, then one gross-error bound of 0.03 / TV 0.10) after float-level noise, the shim stayed byte-identical and no item was involved. CONTEXT: our shim answers HTTP 422 above 40,960 tokens (no truncation); the 15 items of 76,562-81,737 tokens were refused exactly as pre-registered by id and position and are scored wrong; the largest answered item is 3,595 tokens. DETERMINISM: over the first 50 positions, two passes, 6/50 were bit-identical, 0 argmax flips, max probability difference 0.029 (vLLM batching and prefix-cache bf16 noise); the probabilities need not equal the author's or the DI lab's. TRAINING DATA, the author's own statement: the card names no training datasets and warns that some training data carries non-commercial or attribution terms ('check them yourself before commercial use'; not a conclusion about the weights); the training-base revision is undeclared (PEFT revision null), so our base pin is a serving pin, not provenance. The author states it ran the full public Decision Index 0.3 suite (140,620 requests) with the official kit and compared adapters on it; the author's code comments describe benchmark-guided prompt-format changes (our static triage: development exposure, no item-specific rule found); no JevBench use or denial is declared; not independently verified. SELF-REPORTED, unverified: public Decision Index 65.08 (the DI kit rescore is 65.46; the DI lab's Decision Index 0.4.1 full score is 67.57, decision agreement with the submitted rows 99.80% on 831 rows against a 98.5% bar), latency median 139 ms on one RTX PRO 6000. 10 s RULE: max measured decision latency 0.81 s; 0 decisions over 10 s.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 87.43555555555558,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 70.64000000000004
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.43555555555558
      },
      "calibration_gen": {
       "accuracy": 0.801980198019802,
       "cc": 68.26435357016614,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 60.00000000000001,
        "score": 70.24374999999998
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 68.26435357016614
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 88.10313540870452,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 74.96884615384616
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 88.10313540870452
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 98.24777777777778,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 94.74333333333334
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 98.24777777777778
      },
      "jaggedness": {
       "accuracy": 0.7888888888888889,
       "cc": 58.138797836504914,
       "cc_by_type": {
        "choice": 76.90086621751685,
        "noul": 35.13513513513513,
        "score": 62.380392156862754
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.138797836504914
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.77958729004955,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 95.8762886597938,
        "score": 60.21725
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.77958729004955
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 65.07543446244478,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 75.0,
        "score": 44.96857142857144
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 65.07543446244478
      },
      "multilingual": {
       "accuracy": 0.7818181818181819,
       "cc": 62.92410190836234,
       "cc_by_type": {
        "choice": 78.4805999347897,
        "noul": 47.22222222222223,
        "score": 63.0694835680751
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 62.92410190836234
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 82.18677777777778,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 80.0,
        "score": 73.227
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 82.18677777777778
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 87.12279086779087,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 73.38038461538463
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 87.12279086779087
      },
      "usecase": {
       "accuracy": 0.7181818181818181,
       "cc": 58.217503006535956,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 45.83333333333333,
        "score": 55.77996
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 58.217503006535956
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 58.70851037851037,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100.0,
        "score": 14.220769230769214
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 58.70851037851037
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 82.38133333333333,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 47.143999999999984
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 82.38133333333333
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.51293446197256,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 74.43260869565218
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.51293446197256
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 65.27459342023587,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 96.00999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 65.27459342023587
      },
      "en": {
       "accuracy": 0.8496959165942659,
       "cc": 73.91733013139692,
       "cc_by_type": {
        "choice": 84.8427527123174,
        "noul": 72.66187050359711,
        "score": 64.24736717827624
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 73.91733013139692
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 68.17863608738764,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 24.111764705882344
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.17863608738764
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.679927140255,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 96.49333333333333
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.679927140255
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 94.5440350877193,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 83.6321052631579
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 94.5440350877193
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 77.16916666666667,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75.0,
        "score": 56.50750000000001
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 77.16916666666667
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 63.860603496269896,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 5.6114285714285455
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.860603496269896
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 43.283918453799316,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 60.00000000000001,
        "score": 27.04000000000001
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 43.283918453799316
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 10.035661375661372,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 74.55142857142856
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 10.035661375661372
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.2838888888889,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 95.51833333333335
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.2838888888889
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -3.348675324675322,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 15.927999999999997
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 67.13170542635659,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 39.99999999999999,
        "score": 61.395116279069775
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.13170542635659
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 76.00928311187991,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 59.84870967741933
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.00928311187991
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 58.087695035461,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 39.79500000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 58.087695035461
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 63.50320652173914,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50.0,
        "score": 62.24875000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 63.50320652173914
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.80564516129031,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 50.67499999999998
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.80564516129031
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 53.97777777777778,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 28.6
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 53.97777777777778
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8733333333333333,
       "cc": 83.63918176773377,
       "cc_by_type": {
        "choice": 83.63918176773377
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.63918176773377
      },
      "noul": {
       "accuracy": 0.8453333333333334,
       "cc": 69.06666666666668,
       "cc_by_type": {
        "noul": 69.06666666666668
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 69.06666666666668
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 62.66353790613719,
       "cc_by_type": {
        "score": 62.66353790613719
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 62.66353790613719
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 16.118627450980387,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -25.0,
        "score": 21.35588235294116
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 16.118627450980387
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 84.18102564102564,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 52.54307692307694
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 84.18102564102564
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 60.77650406504063,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": 31.10999999999997
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 60.77650406504063
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -27.201396198830412,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -11.42875000000001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.12927536231884,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 73.03999999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.12927536231884
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 33.06406708595387,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -17.160000000000018
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.06406708595387
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 33.46719026071966,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -14.28571428571429,
        "score": 34.29512820512819
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 33.46719026071966
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.090661515931934,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50.0,
        "score": 67.63086956521738
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.090661515931934
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 46.62185185185185,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 21.53222222222221
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 46.62185185185185
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 66.72214273354531,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 46.347727272727276
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 66.72214273354531
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 68.96410774410775,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 29.11454545454546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 68.96410774410775
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 67.53420698924731,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 67.11874999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 67.53420698924731
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 60.68493211061349,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 73.57111111111111
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 60.68493211061349
      },
      "model_routing": {
       "accuracy": 0.8940092165898618,
       "cc": 79.85720542478356,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 100.0,
        "score": 48.92226562500001
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.85720542478356
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 24.02750291671371,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 83.59882352941176
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 24.02750291671371
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 61.275080470442596,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 67.20500000000003
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 61.275080470442596
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 24.474419974886505,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0.0,
        "score": 17.758235294117654
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 24.474419974886505
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 29.065942028985507,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": -33.333333333333336,
        "score": 49.516666666666666
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 29.065942028985507
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.73052631578946,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.19157894736841
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.73052631578946
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.499122489912,
      91.4938757484774
     ],
     "I": [
      66.4121378530749,
      75.44492915004695
     ],
     "capability": [
      77.54868244351084,
      82.87442403920699
     ],
     "composite": {
      "A": [
       42.76962119886615,
       44.17072712154098
      ],
      "B": [
       43.12257834116863,
       45.31558883084057
      ],
      "C": [
       42.76962119886615,
       44.17072712154098
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.33817262545804,
     "capability": 81.3994552562568,
     "intelligence": 71.46073788705554
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8733333333333333,
      "calibration": 90.76921793438983,
      "cc": 83.63918176773377,
      "cc_by_type": {
       "choice": 83.63918176773377
      },
      "cc_tiered": 81.51057544813698,
      "cc_tiered_open": 78.86591280029805,
      "cc_tiered_sealed": 84.1552380959759,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.63918176773377
     },
     "noul": {
      "accuracy": 0.8453333333333334,
      "calibration": 92.36030621057752,
      "cc": 69.06666666666668,
      "cc_by_type": {
       "noul": 69.06666666666668
      },
      "cc_tiered": 70.32483501664956,
      "cc_tiered_open": 75.82125603864733,
      "cc_tiered_sealed": 64.82841399465178,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.06666666666668
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 90.88499373140681,
      "cc": 62.66353790613719,
      "cc_by_type": {
       "score": 62.66353790613719
      },
      "cc_tiered": 62.546803196380104,
      "cc_tiered_open": 64.62896484620214,
      "cc_tiered_sealed": 60.46464154655807,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.66353790613719
     }
    },
    "raw_on_A": {
     "calibration": 88.32281914978606,
     "capability": 79.47232648522377,
     "intelligence": 70.6218338206615
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 39.894548172664095,
    "Emphasis on Accuracy 60:20:20": 42.787874114936656,
    "Emphasis on Cost 20:20:60": 33.82162948100326,
    "Emphasis on Speed 20:60:20": 44.92310669124413,
    "Intelligence only": 48.010802008180605
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ALERIA-ML (GitHub; Aleria AI)",
   "axes": {
    "calibration": 85.70385668097765,
    "cost": 35.82235156694725,
    "intelligence": 68.01802916815697,
    "speed": 86.37984483839423
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06242195896437067,
      "ece_score": 87.51560820712587,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.34873343103085,
      "score_before_time_limit": 87.51560820712587,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.06933333333333336,
      "brier": 0.07996263937935019,
      "ece": 0.0693347575324291,
      "n": 375,
      "score": 85.6736722348821,
      "score_before_time_limit": 86.13304849351418,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9653333333333334,
      "interval90_width": 0.3844444444444446,
      "n": 375,
      "nrps": 0.07381293040469014,
      "score": 85.08916437702001,
      "score_before_time_limit": 85.77536731554437,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.10533986164221125
     }
    },
    "score": 85.70385668097765
   },
   "capability": 76.86094292456731,
   "composite_ci95": {
    "A": [
     30.440000946251576,
     31.551909216464
    ],
    "B": [
     30.783027352582284,
     32.476457138307694
    ],
    "C": [
     30.440000946251576,
     31.551909216464
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7): base string google/gemma-4-31B-it (the stock weights the technique runs on; the author's own provenance file pins the same revision) priced as google/gemma-4-31b-it USD 0.09/M input, USD 0.34/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author server's own usage block, kept unchanged: input_tokens is the SUM of the prompt lengths of every branch the server scored for the item, not the length of one prompt (the server runs several prompts per question: both option orders, the nine-digit rating branches of a two-option or noul question, and a gated second pass of further orders on a question with 3+ options when the first two orders disagree). That is why the total is high: 5,783,944 input tokens over the 1,500 answered items. output_tokens = 0 (prefill and readout of the label tokens, nothing generated). The offline count through a first-party transcription of the author's prompt builder and the pinned tokenizer predicted the server's numbers EXACTLY, item by item: 1,368 items equal the first-pass count and 132 equal first pass plus gated extra, 0 mismatches. K/V-cache reuse inside the server (cached_tokens) is not recorded in the raw rows, so no cache discount is reflected. There were 0 refusals. OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.137802 per 1,000 decisions on the common basis (USD 0.347037 over all 1,500 items) is 2.133x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them); their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.34703663999999995
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.13780181818181816
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Aleria Judge 1.1.1 (ALERIA-ML, inference technique on stock Gemma 4 31B, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.535/h. The author's own `python -m aleria_judge` from the pinned commit on 127.0.0.1:30104 inside a network-none Docker container with the cut proven from inside before the input was read (loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only weights and task input, empty credential placeholders (checked by name in image and container), HF offline flags, trace off, one request at a time. Timed over loopback on the VM. Measured 22 Oct 2026 (VM created 07:57Z, removed 08:23Z) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked locally, on the node and inside the container); determinism 50/50 bit-identical; node input shredded and VM and volume destroyed afterwards (teardown confirmed).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.49470287748886,
    "I_sealed": 65.5413554588251,
    "base": 68.01802916815697,
    "excess": -4.886179351691943,
    "gap": 4.953347418663768,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 79.62086718847044,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 72.31377719182598,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 74.86473429951691,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 70.0,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 56.99850714447922,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.51168140265938,
       "hard": 60.80173971760845,
       "judge": 58.569917341521595,
       "standard": 41.77833957356712
      }
     },
     "sealed|choice": {
      "cc": 79.4026656127611,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 76.06106163987509,
       "judge": 79.72575905974536,
       "standard": 80.06749630879561
      }
     },
     "sealed|noul": {
      "cc": 70.14222113450113,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 89.65517241379311,
       "hard": 63.63636363636365,
       "judge": 72.30769230769232,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 47.07917962921306,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 68.54236024220923,
       "hard": 45.06526528589918,
       "judge": 41.97881382833055,
       "standard": 48.02596671066647
      }
     }
    }
   },
   "jevbench_score": 31.173873132083898,
   "key": "aleria-judge-1.1.1",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (code repo LICENSE, NOTICE and pyproject; the weights are stock google/gemma-4-31B-it, model card apache-2.0, no LICENSE file in the tree; the NVIDIA TensorRT-LLM base image carries NVIDIA's product terms), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ALERIA-ML/aleria-judge@ed2fcb1ac7bef5ef5e392a971dfe4f39ee53080d",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9312320916905444,
    "decisive": 349,
    "decisive_rate": 0.935656836461126,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 111,
   "ranked": true,
   "ranks": {
    "A": 111,
    "B": 104,
    "C": 86,
    "capability": 49
   },
   "repo": "https://github.com/ALERIA-ML/aleria-judge",
   "scores": {
    "A": 31.173873132083898,
    "B": 31.85629092599008,
    "C": 31.173873132083898
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.41476367199984454,
    "p50_s_raw": 0.13238183599992226,
    "p95_s_adjusted": 0.554900102999988,
    "p95_s_raw": 0.202450051499994
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own `aleria_judge` package (ALERIA-ML/aleria-judge, tag v1.1.1 = commit ed2fcb1ac7bef5ef5e392a971dfe4f39ee53080d, tree e7caf1e6182e22ed58e90e749a750a1bbc4e135b, Apache-2.0, 62 files, 60 text files hash-verified by git blob oid; `main` HEAD is the same commit), unmodified: our harness image adds a second layer on top of the author's own Dockerfile (Dockerfile + aleria_judge/**, built `--network none --pull=false`; its `RUN verify_runtime()` step checks the TensorRT-LLM sources), started as the author's `python -m aleria_judge` (the Dockerfile CMD) with the author's own defaults: `JUDGE_ENGINE=lean` (own Triton/FlashInfer token-tree forward, BF16), effort `high` (no options.effort sent), vision tower loaded (`JUDGE_VISION=true`, 560 soft tokens), `JUDGE_CPU_THREADS=8`, plus `JUDGE_MODEL_PATH=/model`, `JUDGE_HOST=127.0.0.1`, `JUDGE_TRACE=false` and `JUDGE_CONTEXT_LENGTH=131072` (the author's submitted 1.1.1 default for the lean engine, config.py:21-24; /health reported differs_from_evaluated {} before and after the sealed run). The author's compose file was not used. Static source review PASS WITH FINDINGS, no blockers (no subprocess, eval/exec, pickle, torch.load, sockets or HTTP clients in the author package; no trust_remote_code; no rule keyed to item text or benchmark names; item text can leave the process only through /v1/trace, which was switched off). DISCLOSED CODE PATTERNS, cleared by the benchmark owner (21 Oct 2026; author code on a rented Verda pod only, network none): (1) the author's default start downloads the 62.6 GB checkpoint from the Hub (aleria_judge/checkpoint.py:98, probe at :90; the compose file makes this the default) - never reached, because JUDGE_MODEL_PATH=/model returns at checkpoint.py:80-83 with a pre-staged, sha256-verified checkpoint, HF offline flags, empty credentials and a network-none container; (2) the author's compose file uses network_mode: host and passes HF_TOKEN through (docker-compose.yml:28,50) - not used; (3) FlashInfer/Triton JIT-compile kernels at first use (lean_core.py:342, @triton.jit) and the NVIDIA image is not source-audited - run with tmpfs scratch inside the network-none container, after a synthetic smoke in the same container shape. Weights: google/gemma-4-31B-it at 842da3794eaa0b77d5f08bae87a17459d91ff475 (stock, not gated), the 9 serving files (62,578,656,403 bytes) fetched by us and sha256-verified on the host against the pinned revision, mounted read-only at /model (a mount deviation accepted by the benchmark owner) and re-verified inside the container before any item was read. Image: NVIDIA nvcr.io/nvidia/tensorrt-llm/release@sha256:124da3f10b73afbf4242f5c68fb04b48736c61fbe662e715202371fd7f046f43 (tag 1.3.0rc29, pulled by digest on the pod before the container network was cut); run image id sha256:f309f6a9dfc5ee6b20c9601614952fd00cbcbcc7456051e902a4aaadeac74fb9. In-container code check: 29 author files identical to the pinned manifest (0 failures); the stock base image's own tensorrt_llm/ tree (1,120 extra files, 153 of them .py, tree sha256 5551af64...98ed5) is admitted only by that pinned tree hash plus a package-identity check, both OK (this allowance was added by us after two pre-sealed attempts stopped at the code check because the stock image ships that source tree; no item was involved). FIRST MEASUREMENT (found by our own intake: open submission, Decision Index 0.4.1 row 13; it replaces the earlier board entry morj-1.0.1 of the same project, formerly named MORJ; neither was on JevBench before). INFERENCE TECHNIQUE ON STOCK WEIGHTS: no fine-tuning and no generated text; the author reads the model's next-token distribution over option labels for several rotations of the option order and combines them (README). SEVERAL PROMPTS PER QUESTION: see cost_basis; the native usage is the sum over all scored branches and was verified item by item. AUTHOR-DEFAULT CONTEXT LIMIT 131,072 (the author's submitted 1.1.1 default for the lean engine; the author's public Decision Index run used 65,536 and states that requests within 65,536 get identical answers at the higher limit, checked on 200 index requests): the server answers HTTP 422 above the limit, never a truncation; no status proxy was used. On this draw there were 0 refusals and 1,500 HTTP 200 answers. The 15 items whose longest branch is 76k-82k tokens (76,595-81,839) were ANSWERED, but each took 24.9-27.8 s, so under the JevBench 10 s rule they count as failed decisions; at the author's public-run limit of 65,536 the same 15 items would have been refused, so either limit scores them as failed. Every other decision took 0.384 s or less (median 0.13 s). BF16, effort high (the author's default) and the vision tower loaded although the items are text only, as in the author's compose defaults. SERIAL: we sent strictly one request at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). The author states that last-bit probability differences are possible on other hardware. METHOD AND TUNING, the author's own statements (docs/method.md, unverified): the Decision Index was also the author's development benchmark and the serving defaults were selected on index runs; the option-count temperature curve (for questions with 3+ options) was fitted on 512 examples from the training splits of BANKING77, CLINC150, WANLI and ARC-Challenge, of which BANKING77, CLINC150 and ARC are also sources of index benchmarks; the author reports that one fit example duplicates a CLINC150 test item up to punctuation and 7 more differ from one by a word or two, and that refitting without them gives the same constants or moves one grid step. We did not check this independently. The prompt layout comes from Mapika/decider, the question list from wcole3/jev-brain. SELF-REPORTED, unverified: public Decision Index 0.3 score 63.43 on H100 PCIe 80 GB and 63.46 on RTX PRO 6000, median / p80 / mean latency 119 / 403 / 320 ms on RTX PRO 6000 (README); the Decision Index lab reports 99.90% decision agreement with the submitted rows on 831 rows (bar 98.5%). HARDWARE: RTX PRO 6000 96 GB, the card the author validated on; latency reflects that card. 10 s RULE: 15 of 1,500 decisions exceeded 10 s (max 27.79 s); no other decision was near the limit.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 76.34111056721059,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 45.689998368298454
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 76.34111056721059
      },
      "calibration_gen": {
       "accuracy": 0.8712871287128713,
       "cc": 77.69535970051153,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 92.0,
        "score": 69.71810455222389
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 77.69535970051153
      },
      "composite_gen": {
       "accuracy": 0.8514851485148515,
       "cc": 77.47093029762064,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 85.18518518518519,
        "score": 63.48776830930274
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 77.47093029762064
      },
      "fairness": {
       "accuracy": 0.8958333333333334,
       "cc": 88.40516683287034,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 84.26311954623004
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 88.40516683287034
      },
      "jaggedness": {
       "accuracy": 0.6444444444444445,
       "cc": 41.748078200203885,
       "cc_by_type": {
        "choice": 68.23869104908566,
        "noul": 35.13513513513513,
        "score": 21.870408416390863
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 41.748078200203885
      },
      "llm_router": {
       "accuracy": 0.9137380191693291,
       "cc": 82.52291836464019,
       "cc_by_type": {
        "choice": 92.07537201725808,
        "noul": 93.81443298969072,
        "score": 61.67895008697178
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 82.52291836464019
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 41.005561940426084,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 25.0,
        "score": 22.75895386251534
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 41.005561940426084
      },
      "multilingual": {
       "accuracy": 0.7563636363636363,
       "cc": 62.40099059114015,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 61.111111111111114,
        "score": 54.45834256645017
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 62.40099059114015
      },
      "negation_pair_gen": {
       "accuracy": 0.8604651162790697,
       "cc": 67.74274557620764,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 49.894903395289546
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 67.74274557620764
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 84.94291126482074,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 69.84374880947725
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.94291126482074
      },
      "usecase": {
       "accuracy": 0.759090909090909,
       "cc": 64.88213894006674,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 70.83333333333333,
        "score": 44.646416820200216
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.88213894006674
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 27.79711937467592,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": -47.561022828353195
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.79711937467592
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 82.047283199588,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 66.87917678770407
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 82.047283199588
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.87936515069799,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 78.5319007618285
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.87936515069799
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 53.368376750417035,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 93.6246833238768
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 53.368376750417035
      },
      "en": {
       "accuracy": 0.8184187662901824,
       "cc": 69.53360743701114,
       "cc_by_type": {
        "choice": 81.54769895412551,
        "noul": 76.25899280575538,
        "score": 50.79413055115255
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 69.53360743701114
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.73821861479911,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 19.79051228811676
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.73821861479911
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 64.81788141933258,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 93.90719617056605
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 64.81788141933258
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 80.95313634625022,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 64.03587962698593
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 80.95313634625022
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 67.89002872687081,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 50.0,
        "score": 53.67008618061242
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 67.89002872687081
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 57.83441482601925,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -12.467137439323395
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 57.83441482601925
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 49.453900162875755,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": 16.95582280792829
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 49.453900162875755
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 77.85294894387718,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0,
        "score": 78.003291276076
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 77.85294894387718
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.37352154231557,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 92.78723129361339
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.37352154231557
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 36.12208525541918,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 26.980974380976164
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 36.12208525541918
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 65.24982382273507,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 60.00000000000001,
        "score": 50.984342394993185
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 65.24982382273507
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 68.56571063038,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 48.1249456801795
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 68.56571063038
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 45.97867858809504,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 28.999865551519164
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 45.97867858809504
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 63.77894721198077,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": 34.815102505507525
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 63.77894721198077
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.10560716642341,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 51.57488601539926
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.10560716642341
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 62.37911574389773,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 53.80401389835985
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 62.37911574389773
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8453333333333334,
       "cc": 80.02257984270653,
       "cc_by_type": {
        "choice": 80.02257984270653
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 80.02257984270653
      },
      "noul": {
       "accuracy": 0.8666666666666667,
       "cc": 73.33333333333334,
       "cc_by_type": {
        "noul": 73.33333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 73.33333333333334
      },
      "score": {
       "accuracy": 0.672,
       "cc": 50.375231807687534,
       "cc_by_type": {
        "score": 50.375231807687534
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.375231807687534
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.8181818181818182,
       "cc": 63.70973568583323,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 75.0,
        "score": 32.1292070574997
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 63.70973568583323
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 73.25072028446563,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 19.752160853396884
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 73.25072028446563
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 59.529818099139305,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 11.109779500669948
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 59.529818099139305
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 25.070612463705256,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": 12.053942654273664
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 25.070612463705256
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 57.83185504321008,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 69.14773904267372
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 57.83185504321008
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": 16.455332023352472,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -33.65287185447088
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 16.455332023352472
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 48.0998002228388,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 21.050100948628447
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 48.0998002228388
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 22.939882495237857,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50.0,
        "score": 50.178532503135166
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 22.939882495237857
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 31.04613908628008,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": -4.361582741159764
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 31.04613908628008
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 73.53099270250745,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100.0,
        "score": 46.28890958503842
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 73.53099270250745
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 62.74756127877338,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 10.464906058542367
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 62.74756127877338
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 39.69115230243343,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 50.25625260622504
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 39.69115230243343
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 72.37465401331396,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 57.12396203994186
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.37465401331396
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 82.39917006452004,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100.0,
        "score": 53.04166603771596
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.39917006452004
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 55.11651371368234,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0.0,
        "score": 76.86585592031764
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 55.11651371368234
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.88731244243987,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 64.75478783799744
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.88731244243987
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 29.047727091875924,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": 16.699831521933174
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 29.047727091875924
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 45.371680903544984,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 46.259970246866835
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 45.371680903544984
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 97.62489117390125,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 92.87467352170374
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.62489117390125
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.5578208281358,
      87.22858339554696
     ],
     "I": [
      62.68314091383449,
      72.03922327034007
     ],
     "capability": [
      72.83738772532513,
      78.93901389771851
     ],
     "composite": {
      "A": [
       30.440000946251576,
       31.551909216464
      ],
      "B": [
       30.783027352582284,
       32.476457138307694
      ],
      "C": [
       30.440000946251576,
       31.551909216464
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.70385668097765,
     "capability": 76.86094292456731,
     "intelligence": 68.01802916815697
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8453333333333334,
      "calibration": 86.34873343103085,
      "cc": 80.02257984270653,
      "cc_by_type": {
       "choice": 80.02257984270653
      },
      "cc_tiered": 79.51176640061577,
      "cc_tiered_open": 79.62086718847044,
      "cc_tiered_sealed": 79.4026656127611,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 80.02257984270653
     },
     "noul": {
      "accuracy": 0.8666666666666667,
      "calibration": 85.6736722348821,
      "cc": 73.33333333333334,
      "cc_by_type": {
       "noul": 73.33333333333334
      },
      "cc_tiered": 72.50347771700902,
      "cc_tiered_open": 74.86473429951691,
      "cc_tiered_sealed": 70.14222113450113,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 73.33333333333334
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 85.08916437702001,
      "cc": 50.375231807687534,
      "cc_by_type": {
       "score": 50.375231807687534
      },
      "cc_tiered": 52.03884338684614,
      "cc_tiered_open": 56.99850714447922,
      "cc_tiered_sealed": 47.07917962921306,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.375231807687534
     }
    },
    "raw_on_A": {
     "calibration": 85.96699197533792,
     "capability": 76.92406180470816,
     "intelligence": 67.8811316340784
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 28.41422995653169,
    "Emphasis on Accuracy 60:20:20": 30.7001814847762,
    "Emphasis on Cost 20:20:60": 23.32627553027168,
    "Emphasis on Speed 20:60:20": 33.1810394815314,
    "Intelligence only": 34.91340601876365
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Arcbright (arcbright on Hugging Face)",
   "axes": {
    "calibration": 91.63064878963459,
    "cost": 49.70469532224864,
    "intelligence": 71.94618834920504,
    "speed": 89.86892425290594
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.045599768064632624,
      "ece_score": 90.88004638707348,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.88004638707348,
      "score_before_time_limit": 90.88004638707348,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.07020344233521425,
      "ece": 0.03816948771665763,
      "n": 373,
      "score": 92.36610245666847,
      "score_before_time_limit": 92.36610245666847,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9811827956989247,
      "interval90_width": 0.39986559139784944,
      "n": 372,
      "nrps": 0.06587467173838109,
      "score": 91.64579752516181,
      "score_before_time_limit": 91.64579752516181,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05060468887919142
     }
    },
    "score": 91.63064878963459
   },
   "capability": 81.78841856941982,
   "composite_ci95": {
    "A": [
     69.13383978576802,
     71.18740400978247
    ],
    "B": [
     68.76871271910235,
     71.74019814422863
    ],
    "C": [
     69.13383978576802,
     71.18740400978247
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7): base string google/gemma-4-31B-it (the card's base_model) priced as google/gemma-4-31b-it USD 0.09/M input, USD 0.34/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author's own server usage block (input_tokens = tokens of the rendered chat prompt incl. system prompt, options and the answer prefill; output_tokens = 1, one vLLM forward pass per answered single-question row, nothing generated). The offline count through the author's render.py and the pinned tokenizer predicted the server's numbers EXACTLY, item by item: 777,809 input tokens over the 1,485 answered items, and 15 HTTP 422 refusals at the pre-registered ids and positions.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.047005139999999994
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04747993939393939
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Arcbright Elf 1 31B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.525/h. The author's own `python3 -m elf_serve` from the pinned commit on 127.0.0.1:8000 inside a network-none Docker container with the cut proven from inside (loopback only, no default route, DNS and TCP fail), offline and credential-free, one request at a time. Timed over loopback on the VM. Measured 22 Oct 2026.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 74.68088774011248,
    "I_sealed": 69.2114889582976,
    "base": 71.94618834920504,
    "excess": -4.370127988540823,
    "gap": 5.469398781814888,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 76.10361879897965,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 64.40342781806197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 91.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 100.0,
       "judge": 70.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 56.9390444213578,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.89638859502351,
       "hard": 64.43144230077161,
       "judge": 48.69808364892321,
       "standard": 48.83701773434921
      }
     },
     "sealed|choice": {
      "cc": 80.12686413307563,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 75.02023823291314,
       "judge": 79.72575905974536,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 77.86377492810843,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 70.9090909090909,
       "judge": 81.53846153846153,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 49.643827813708725,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 73.05883560337949,
       "hard": 46.65352270756829,
       "judge": 48.28621943708376,
       "standard": 45.95334669609168
      }
     }
    }
   },
   "jevbench_score": 70.51330070509262,
   "key": "arcbright-elf-1-31b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card, root LICENSE and serve/LICENSE; base google/gemma-4-31B-it under the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "arcbright/elf-1-31b@8c7799280c8c04a3802af1feeb3a1ad40d6c4eeb",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9115281501340483,
    "decisive": 373,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 25,
   "ranked": true,
   "ranks": {
    "A": 25,
    "B": 19,
    "C": 23,
    "capability": 20
   },
   "repo": "https://huggingface.co/arcbright/elf-1-31b",
   "scores": {
    "A": 70.51330070509262,
    "B": 70.62963891559733,
    "C": 70.51330070509262
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27325655800009374,
    "p50_s_raw": 0.06162827900004686,
    "p95_s_adjusted": 0.37716986049996426,
    "p95_s_raw": 0.11358493024998212
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own `elf_serve` package at the pinned revision 8c7799280c8c04a3802af1feeb3a1ad40d6c4eeb, unmodified: serve/elf_serve/{__init__,__main__,joint,render,server,settings,worker}.py, byte-identical (sha256, 7/7) to the arcbright/elf-1-12b serve files measured 20 Oct; the author's serve/SHA256SUMS verifies 11/11 inside the container. Started as `python3 -m elf_serve` with ELF_MODEL pointed at the commit-pinned local snapshot and the repo-root decision_config.json (the configuration of the author's Decision Index run: hybrid-v1, online FP8 from the bf16 checkpoint, max_model_len 32,768, temperatures scalar 2.186 / choice 1.7509 / noul 1.1758 / score 2.1257, noul_commit [0.199, 0.801]); ELF_CONFIG was never set, so serve/configs/elf-1-31b-prod.json was not used. Stock digest-pinned vllm/vllm-openai:v0.30.0 image plus the same hash-pinned wheels as the 12B row (offline --no-deps install). The two audited patterns (two read-only constant nvidia-smi queries in settings.py; a snapshot_download fallback unreachable as measured, network-none container) ride along unchanged: The benchmark owner cleared these exact bytes for this one measurement before the run. Weights: the commit-pinned snapshot, every file sha256-verified on the host and again in the container. FIRST MEASUREMENT (found by our own intake: Hugging Face decision-model tag / Decision Index). AUTHOR-DEFAULT CONTEXT CAP: the shipped decision_config.json caps a request row at 32,768 tokens; the author's server returns a real HTTP 422 above it, never a truncation. The 15 items of 76k-82k tokens were refused exactly as pre-registered and are scored wrong; the largest answered item is 1,828 tokens. ONLINE FP8: the author's Decision Index configuration quantises the bf16 checkpoint to FP8 at load time; this row is that configuration. NOUL COMMIT: the shipped config moves an interior noul probability to the nearer of 0.199 / 0.801 (author default, kept). SAME CODE AS ELF 1 12B: the seven serve files are byte-identical to the measured arcbright-elf-1-12b row. AUDITED CODE PATTERNS: see code_revision. TRAINING DATA, the author's own statement: one LoRA on the attention layers merged into bf16, 38,000 rows from public datasets incl. ZefanCai/Open-Jev and Open-Jev-v1.1 plus own generators, including the train splits of six Decision Index benchmarks (HoVer, WinoGrande, When2Call, GSM8K, BANKING77, CLINC150); the author states it screened for overlap with Decision Index test items - not independently verified here. SELF-REPORTED, unverified: 84 ms median latency (FP8, one RTX PRO 6000) on the author's card. 10 s RULE: max measured decision latency 0.24 s; no decision near the limit.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 57.204585699967765,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": -20.052909566763375
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 57.204585699967765
      },
      "calibration_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 67.97203660818023,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 60.00000000000001,
        "score": 59.822790630479176
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 67.97203660818023
      },
      "composite_gen": {
       "accuracy": 0.8712871287128713,
       "cc": 80.33590291986127,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 85.18518518518519,
        "score": 62.326588615049026
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 80.33590291986127
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 95.78198861246427,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 87.34596583739285
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 95.78198861246427
      },
      "jaggedness": {
       "accuracy": 0.6777777777777778,
       "cc": 55.1268918875868,
       "cc_by_type": {
        "choice": 56.689124157844084,
        "noul": 67.56756756756756,
        "score": 41.12398393734876
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 55.1268918875868
      },
      "llm_router": {
       "accuracy": 0.8945686900958466,
       "cc": 80.79446793487303,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 95.8762886597938,
        "score": 56.01666872411562
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.79446793487303
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 54.17489074061117,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75.0,
        "score": 24.63807428368917
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 54.17489074061117
      },
      "multilingual": {
       "accuracy": 0.7890909090909091,
       "cc": 69.06341467802686,
       "cc_by_type": {
        "choice": 76.5242908379524,
        "noul": 77.77777777777777,
        "score": 52.8881754183504
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.06341467802686
      },
      "negation_pair_gen": {
       "accuracy": 0.9069767441860465,
       "cc": 76.81877153861538,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 80.0,
        "score": 57.12298128251282
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.81877153861538
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 80.21167381519848,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 55.650036460610465
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.21167381519848
      },
      "usecase": {
       "accuracy": 0.7318181818181818,
       "cc": 65.92152623993582,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 75.0,
        "score": 49.72536303353292
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 65.92152623993582
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 24.201077194803236,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -20.25391127273315
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.201077194803236
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 81.72251651210776,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 45.16754953632326
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 81.72251651210776
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.17845480227358,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 74.46456794664377
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.17845480227358
      },
      "el": {
       "accuracy": 0.8,
       "cc": 76.81907343233173,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 97.31010670295426
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 76.81907343233173
      },
      "en": {
       "accuracy": 0.8192875760208514,
       "cc": 71.79392556276699,
       "cc_by_type": {
        "choice": 80.66901795194102,
        "noul": 81.29496402877699,
        "score": 53.417794707582935
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.79392556276699
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 67.4304993100405,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 21.867354373840953
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.4304993100405
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 64.6764267214245,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 93.48283207684182
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 64.6764267214245
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 81.07498866843883,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 64.4014365935518
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 81.07498866843883
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 72.81153344833051,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0,
        "score": 18.4346003449915
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 72.81153344833051
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 59.19215964961905,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 5.635715114094886
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 59.19215964961905
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 50.387000946248946,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 80.0,
        "score": -0.24487484195212517
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 50.387000946248946
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 92.19353263244655,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 76.58059789733964
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 92.19353263244655
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.70832869724404,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 93.7916527583988
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.70832869724404
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 71.38116254353474,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 100.0,
        "score": 40.117513604630204
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 71.38116254353474
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 74.49489472134356,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80.0,
        "score": 51.1021196274247
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 74.49489472134356
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.77618028751725,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 44.149401204331326
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.77618028751725
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 49.68215630779286,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 14.578383816995565
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 49.68215630779286
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 66.88714612791158,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": 44.13969925329998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 66.88714612791158
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.14435815588773,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 63.69113898379224
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.14435815588773
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 76.32837381348317,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 28.98512144044949
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 76.32837381348317
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8453333333333334,
       "cc": 80.02257984270653,
       "cc_by_type": {
        "choice": 80.02257984270653
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 80.02257984270653
      },
      "noul": {
       "accuracy": 0.9066666666666666,
       "cc": 81.33333333333333,
       "cc_by_type": {
        "noul": 81.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 81.33333333333333
      },
      "score": {
       "accuracy": 0.656,
       "cc": 52.04184226648492,
       "cc_by_type": {
        "score": 52.04184226648492
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 52.04184226648492
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7727272727272727,
       "cc": 59.115790129793766,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 75.0,
        "score": 34.34737038938128
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 59.115790129793766
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 78.39539955294666,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 35.18619865883996
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 78.39539955294666
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 50.139870119207636,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": -17.060064439125068
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 50.139870119207636
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 13.160050611862175,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -0.28593003575849707
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 13.160050611862175
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 61.53698928966147,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 65.7703881588395
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 61.53698928966147
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 52.01161833869051,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100.0,
        "score": 6.34932042487657
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 52.01161833869051
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 47.27061808024164,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 28.3664760894644
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 47.27061808024164
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 39.03343953882915,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0.0,
        "score": 48.459203633909055
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 39.03343953882915
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 59.09317237442428,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100.0,
        "score": 39.77951712327283
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 59.09317237442428
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 69.13866057105047,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100.0,
        "score": 45.95987892942547
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 69.13866057105047
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 66.47674606607687,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 21.652460420452833
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.47674606607687
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 49.12803388587864,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 44.15829520602301
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 49.12803388587864
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.9448826497521,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 58.35096272852691
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.9448826497521
      },
      "model_routing": {
       "accuracy": 0.880184331797235,
       "cc": 79.17182117069517,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100.0,
        "score": 44.528450525072486
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.17182117069517
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 52.43173568281341,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0.0,
        "score": 68.81152182771086
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 52.43173568281341
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 60.80044195437011,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 65.78108445178256
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 60.80044195437011
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 56.63512118362161,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100.0,
        "score": -0.5379862028297433
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 56.63512118362161
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 50.0779780831841,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 45.88610816259578
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 50.0779780831841
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.615459389366,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.84637816809803
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.615459389366
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.37717693640397,
      92.31194864397001
     ],
     "I": [
      68.15049227545063,
      75.01758533219932
     ],
     "capability": [
      78.3609233982891,
      83.20686623472555
     ],
     "composite": {
      "A": [
       69.13383978576802,
       71.18740400978247
      ],
      "B": [
       68.76871271910235,
       71.74019814422863
      ],
      "C": [
       69.13383978576802,
       71.18740400978247
      ]
     },
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.63064878963459,
     "capability": 81.78841856941982,
     "intelligence": 71.94618834920504
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8453333333333334,
      "calibration": 90.88004638707348,
      "cc": 80.02257984270653,
      "cc_by_type": {
       "choice": 80.02257984270653
      },
      "cc_tiered": 78.11524146602764,
      "cc_tiered_open": 76.10361879897965,
      "cc_tiered_sealed": 80.12686413307563,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 80.02257984270653
     },
     "noul": {
      "accuracy": 0.9066666666666666,
      "calibration": 92.36610245666847,
      "cc": 81.33333333333333,
      "cc_by_type": {
       "noul": 81.33333333333333
      },
      "cc_tiered": 84.43188746405421,
      "cc_tiered_open": 91.0,
      "cc_tiered_sealed": 77.86377492810843,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 81.33333333333333
     },
     "score": {
      "accuracy": 0.656,
      "calibration": 91.64579752516181,
      "cc": 52.04184226648492,
      "cc_by_type": {
       "score": 52.04184226648492
      },
      "cc_tiered": 53.291436117533266,
      "cc_tiered_open": 56.9390444213578,
      "cc_tiered_sealed": 49.643827813708725,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.04184226648492
     }
    },
    "raw_on_A": {
     "calibration": 89.21683012213103,
     "capability": 80.3975934709685,
     "intelligence": 71.57835681980598
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.66930257098518,
    "Emphasis on Accuracy 60:20:20": 67.73847613816869,
    "Emphasis on Cost 20:20:60": 57.86994742512285,
    "Emphasis on Speed 20:60:20": 73.31023307695475,
    "Intelligence only": 71.098856133272
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "sanghwa-na (Hugging Face; code foxl-ai/bobcat on GitHub)",
   "axes": {
    "calibration": 87.62628477134197,
    "cost": 49.86924343971057,
    "intelligence": 68.0935397642001,
    "speed": 89.29862693377845
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04389435292018364,
      "ece_score": 91.22112941596328,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 90.0048476904171,
      "score_before_time_limit": 91.22112941596328,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.07999999999999996,
      "brier": 0.08188541684317341,
      "ece": 0.0771803607955985,
      "n": 375,
      "score": 84.11292022572896,
      "score_before_time_limit": 84.56392784088031,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.29177777777777764,
      "n": 375,
      "nrps": 0.06991851787155044,
      "score": 88.7610863978799,
      "score_before_time_limit": 89.47690161076602,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.07027172495656454
     }
    },
    "score": 87.62628477134197
   },
   "capability": 77.85991226777104,
   "composite_ci95": {
    "A": [
     67.66582609883011,
     70.40896775459174
    ],
    "B": [
     66.72253561067964,
     70.64201070858508
    ],
    "C": [
     67.66582609883011,
     70.40896775459174
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string google/gemma-4-31B-it (the card's base_model) priced as google/gemma-4-31b-it USD 0.09/M input, USD 0.34/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's own server: input_tokens = the billed count (state plus question; the author's host template of 83 tokens is excluded and a shared state is counted once; the x-bobcat-processed-tokens header is billed + 83 and is NOT used), output_tokens = 0 on every row (the server never generates text). Our offline count through a first-party transcription of the author's prompt compiler and the pinned tokenizer predicted the server's numbers EXACTLY, item by item: 1,976,831 input tokens over the 1,500 answered items (0 item differences), and 0 refusals as pre-registered (all 1,500 items, including the 15 very long ones of 76,000-82,000 tokens, are inside the author's 98,303-token limit). The 15 very long items count for no row's cost on the common 1,485-item cost basis (v1.6.30). COST ON THE COMMON BASIS: USD 0.046884 per 1,000 decisions on 1,485 items, 0.726 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.11860985999999998
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0468840606060606
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bobcat 1.3 (Gemma 4 31B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. The authors' flash_server (vLLM engine, online FP8, 127.0.0.1:8000) and the 18 hash-pinned author files run in ONE docker container with network none, served at --max-model-len 98368 with --max-num-seqs 64; the driver sent requests one at a time over loopback on the VM. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 06:47-07:12Z; measurement 855 s, total 1,475 s; scored run 673 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:12Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 71.27885942153964,
    "I_sealed": 64.90822010686055,
    "base": 68.0935397642001,
    "excess": -3.4688874556766294,
    "gap": 6.370639314679082,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 74.33918859210414,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 67.9144385026738,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 81.04347826086956,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 58.45391141164524,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.90713302535477,
       "hard": 68.44215068172673,
       "judge": 54.57383133604144,
       "standard": 39.070942178033185
      }
     },
     "sealed|choice": {
      "cc": 78.21398393688209,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 76.58147334335607,
       "judge": 74.43682664054849,
       "standard": 81.01666315123391
      }
     },
     "sealed|noul": {
      "cc": 66.22701539325317,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 61.81818181818181,
       "judge": 60.00000000000001,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 50.28366099044642,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.49288080741778,
       "hard": 49.582586651545235,
       "judge": 45.11116758489915,
       "standard": 50.839939868083995
      }
     }
    }
   },
   "jevbench_score": 69.38453883513192,
   "key": "bobcat-1.3",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (model card and Hub metadata; base google/gemma-4-31B-it: apache-2.0 in its card metadata with a link to the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "sanghwa-na/bobcat-1.3@4f25c70ece312ec4e74c47f1c5264cb2aeea956a",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.936046511627907,
    "decisive": 344,
    "decisive_rate": 0.9222520107238605,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 33,
   "ranked": true,
   "ranks": {
    "A": 33,
    "B": 26,
    "C": 28,
    "capability": 41
   },
   "repo": "https://huggingface.co/sanghwa-na/bobcat-1.3",
   "scores": {
    "A": 69.38453883513192,
    "B": 69.04882890702815,
    "C": 69.38453883513192
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2878270249999787,
    "p50_s_raw": 0.06891351249998934,
    "p95_s_adjusted": 0.4083247809998397,
    "p95_s_raw": 0.12916239049991987
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: foxl-ai/bobcat at 538169365291d7c46d2c116f8ee09ad9f4b3ac5e (5 commits past the previously reviewed 43387a09, +6,032/-45 lines over 34 files, none in the serving path: the 18 serving modules are byte-identical at both commits). Only those 18 files (src/bobcat/*.py, manifest sha256 71dad08892d1e489b312bcd3fffa8a3bcef529181ad1db16deb9c04610d4ff83) were shipped to the VM, read-only at /code/src, hash-verified; every other module of the repo raises ImportError; no git clone, no pip install of the author's package. Command (the 1.3 card's Quickstart plus --identifiers): python3 -m bobcat.flash_server --engine vllm --model /models/bobcat-1.3 --compiler-model /models/bobcat-1.3/compiler --quantization fp8 --temperature choice=1.007793,noul=0.445449,score=1.023435 --name bobcat-1.3 --release-date 2026-10-11 --max-num-seqs 64 --max-model-len 98368 --schedule all --engine-arg max_num_batched_tokens=16384 --engine-arg attention_backend=TRITON_ATTN --identifiers /models/bobcat-1.3/bobcat-identifiers.json --host 127.0.0.1 --port 8000 --local (online FP8 of the BF16 weights, the author's recommended path; no kv_cache_dtype override). Image: official vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 plus ONE hash-pinned wheel (scipy 1.15.3), built offline on the VM (image sha256:42b2ad6131642a63abe73bb9c7fa0690ae47356e1be2bb39988971ef5b00d742); vllm 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, tokenizers 0.23.2. The submitted code was reviewed and cleared for this run because the unchanged serving modules contain dormant hooks (DISCLOSED CODE PATTERNS, none reachable in our setup: a Hub downloader and a model-provided-code exec_module fallback in student_readout.py:21,57-63,334-346 (the model repo has no encoding/ directory and ships a chat template, the container has no network); an env read of BOBCAT_EDGE_SECRET at flash_server.py:87 (empty, as --local allows); boto3 at checkpoints.py:102, hf_hub_download at corpus.py:69, httpx at glm_readout.py:191 and torch.load(weights_only=True) at serve.py:33, all training or command-line paths not reachable from flash_server). The source changes and runner were reviewed before the run, findings were fixed and the fixes rechecked. Weights: sanghwa-na/bobcat-1.3 at 4f25c70ece312ec4e74c47f1c5264cb2aeea956a (Apache-2.0, public, not gated, 22 files, BF16 62,546,338,248 weight bytes; the weight average of three merged rank-16 LoRA arms on google/gemma-4-31B-it), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file (MODEL_MANIFEST.json sha256 882e017a6873c6e312277f39db1c91795f440f79a7dfa00e08258a5b31b694f2). FIRST MEASUREMENT (found by our own intake; Author submission, a submission by the authors; our triage treats it as superseding their Bobcat 1.2 submission author submission; the earlier row bobcat-flash-1.2 is a different model). AUTHOR'S OWN SERVER, on our hardware: the row measures sanghwa-na/bobcat-1.3 served by foxl-ai/bobcat as the card's Quickstart documents, in online FP8, on one RTX PRO 6000 (not the author's B200). 10 s RULE: 15 decisions took longer than 10 s (35-38 s each, max 38.5 s) and are scored wrong under the rule in the v1.7 share view (the frozen v1.6 and v1.7-off views do not apply the 10 s rule); they are exactly the 15 standard long positions (76,000-82,000 tokens), consistent with a cold prefill of the whole state (in the smoke a 78.5k-token request took 34.8 s cold and 0.12 s warm); no sealed item was repeated warm. The long items are answered in full (never truncated, never refused). The other 1,485 decisions were fast (p50 0.069 s, p95 0.129 s; the 99th percentile over all 1,500 is 35.1 s only because of these 15). DETERMINISM: first 50 positions run twice, 48/50 bit-identical, 0 argmax flips, max probability difference 0.0012 (online FP8 server). OPEN PARITY RISK: the 1.3 release manifest binds a student_readout.py (sha256 e5597653...) that was never published; the public file (f8d3ceb8e1f33509..., the only version ever pushed, also bound by the 1.2 manifests) is what the card's Quickstart installs and what we ran, so parity with the author's own 1.3 evaluation is unproven. AUTHOR EXPOSURE (card, self-reported, not verified): the JevBench public 231 was evaluated three times by the authors (unofficial Capability 77.2); the Decision Index public half was used for candidate selection; the Choice and Score temperatures were fitted on LocalLLaMA/typed-decisions even test ids and the Noul temperature on a held-out JevBench-format split. The authors' delta audit code drops training rows that overlap the public 231; we did not audit it, ran no overlap check and cannot rule out other overlap. The fitted temperatures (choice 1.007793, noul 0.445449, score 1.023435) are part of the served configuration. The card's latency numbers (61.4 ms on RTX PRO 6000 for Bobcat 1.2, 28.6 ms on B200 for 1.3) are the author's and do not transfer. TOKEN COUNTING: our counter was validated by one native datum from the earlier bobcat-flash-1.2 run (105) and then by the sealed item-by-item comparison, which matched exactly; the image used tokenizers 0.23.2, the offline counter 0.22.2. No outside request for this model; priced as an estimate from the base model, not from any tariff of the authors.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 74.23969124852903,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 39.38574041225378
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 74.23969124852903
      },
      "calibration_gen": {
       "accuracy": 0.6336633663366337,
       "cc": 52.81534073389674,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 52.0,
        "score": 54.1660646195057
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 52.81534073389674
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 85.33266164130478,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 92.59259259259258,
        "score": 73.16148989229738
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.33266164130478
      },
      "fairness": {
       "accuracy": 0.9583333333333334,
       "cc": 92.62055492372407,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 86.66666666666667,
        "score": 91.19499810450556
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 92.62055492372407
      },
      "jaggedness": {
       "accuracy": 0.6055555555555555,
       "cc": 43.755252658109974,
       "cc_by_type": {
        "choice": 55.24542829643889,
        "noul": 51.35135135135136,
        "score": 24.66897832653968
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 43.755252658109974
      },
      "llm_router": {
       "accuracy": 0.9137380191693291,
       "cc": 83.06639153155028,
       "cc_by_type": {
        "choice": 92.86783481553225,
        "noul": 93.81443298969072,
        "score": 62.51690678942784
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.06639153155028
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 52.90360670916882,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 50.0,
        "score": 33.45308816874354
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 52.90360670916882
      },
      "multilingual": {
       "accuracy": 0.7818181818181819,
       "cc": 63.96786213240862,
       "cc_by_type": {
        "choice": 74.5679817411151,
        "noul": 55.55555555555556,
        "score": 61.7800491005552
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.96786213240862
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 78.96146717862894,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 80.0,
        "score": 63.55106820255351
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.96146717862894
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 84.6306841840915,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 65.90406456428653
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.6306841840915
      },
      "usecase": {
       "accuracy": 0.7136363636363636,
       "cc": 59.83498229574659,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 58.33333333333333,
        "score": 48.13239786763192
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.83498229574659
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 37.142210792909026,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": -0.4781295260348317
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 37.142210792909026
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 58.609331259061015,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 63.23198763278983
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 58.609331259061015
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.29697354232822,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 85.78472593671917
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.29697354232822
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.37645472611915,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.31558391764983
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.37645472611915
      },
      "en": {
       "accuracy": 0.8001737619461338,
       "cc": 68.78566887480646,
       "cc_by_type": {
        "choice": 78.91165594757203,
        "noul": 74.82014388489209,
        "score": 52.62520679195526
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 68.78566887480646
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 71.02044939362587,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 22.849276402737296
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.02044939362587
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.32391955062916,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 98.42531056445578
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.32391955062916
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 78.71231633263812,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 78.48989017438495
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 78.71231633263812
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 74.23237617511566,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75.0,
        "score": 47.697128525346976
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 74.23237617511566
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 62.194486664910634,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 0.6130780773507594
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 62.194486664910634
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 45.72426237348003,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": 5.766909439741108
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.72426237348003
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 47.59296668875374,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 87.22334451070569
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 47.59296668875374
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.10819870917948,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.9912627942051
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.10819870917948
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 24.413351253396552,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 32.54741306754896
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 24.413351253396552
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 63.418062521802085,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 57.871623028800265
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 63.418062521802085
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 69.53317456601008,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 51.027337487069715
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 69.53317456601008
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 47.301971952842074,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 32.96974564576027
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 47.301971952842074
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 76.0081598974826,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 49.763610127230415
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 76.0081598974826
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.83303262949057,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 62.757162404600784
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.83303262949057
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 71.27600077139918,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 13.82800231419752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 71.27600077139918
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.832,
       "cc": 78.30038844983639,
       "cc_by_type": {
        "choice": 78.30038844983639
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 78.30038844983639
      },
      "noul": {
       "accuracy": 0.8586666666666667,
       "cc": 71.73333333333333,
       "cc_by_type": {
        "noul": 71.73333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 71.73333333333333
      },
      "score": {
       "accuracy": 0.6693333333333333,
       "cc": 53.323643295776094,
       "cc_by_type": {
        "score": 53.323643295776094
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 53.323643295776094
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 48.21772340389142,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 25.0,
        "score": 35.653170211674244
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 48.21772340389142
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 65.04142526820455,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 17.346498026835878
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 65.04142526820455
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 62.52068687198607,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100.0,
        "score": 3.822223217584231
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.52068687198607
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": 11.507558082242374,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -5.243407624617902
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 11.507558082242374
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 56.64084608163887,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 65.57471215796006
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 56.64084608163887
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 50.05850231278627,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 66.66666666666667,
        "score": 46.401922032698415
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 50.05850231278627
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 41.44357485132011,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 39.456774974128386
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 41.44357485132011
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 28.0719633890198,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50.0,
        "score": 65.574775184481
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 28.0719633890198
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 49.2441230583897,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100.0,
        "score": 10.232369175169087
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 49.2441230583897
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 65.63025255294463,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100.0,
        "score": 35.43465487510799
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.63025255294463
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 49.11013862345109,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 3.4027672915419194
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 49.11013862345109
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 59.40395296198992,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 42.727987918227825
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 59.40395296198992
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 75.06726419420765,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 65.20179258262291
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 75.06726419420765
      },
      "model_routing": {
       "accuracy": 0.8986175115207373,
       "cc": 81.38800385735159,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100.0,
        "score": 51.17699858504179
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.38800385735159
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 23.34421860436829,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 81.5489705923755
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 23.34421860436829
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.85494744780317,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 64.65769285408733
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.85494744780317
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 48.05391633532412,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50.0,
        "score": 23.718399252277777
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 48.05391633532412
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 36.046095497196575,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 51.61654736115495
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 36.046095497196575
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.66123219687104,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.98369659061314
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.66123219687104
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.16269651951559,
      88.93073363136115
     ],
     "I": [
      63.4644225486904,
      72.14683687162247
     ],
     "capability": [
      74.05948716208472,
      79.9460707309049
     ],
     "composite": {
      "A": [
       67.66582609883011,
       70.40896775459174
      ],
      "B": [
       66.72253561067964,
       70.64201070858508
      ],
      "C": [
       67.66582609883011,
       70.40896775459174
      ]
     },
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.62628477134197,
     "capability": 77.85991226777104,
     "intelligence": 68.0935397642001
    },
    "per_type": {
     "choice": {
      "accuracy": 0.832,
      "calibration": 90.0048476904171,
      "cc": 78.30038844983639,
      "cc_by_type": {
       "choice": 78.30038844983639
      },
      "cc_tiered": 76.27658626449312,
      "cc_tiered_open": 74.33918859210414,
      "cc_tiered_sealed": 78.21398393688209,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 78.30038844983639
     },
     "noul": {
      "accuracy": 0.8586666666666667,
      "calibration": 84.11292022572896,
      "cc": 71.73333333333333,
      "cc_by_type": {
       "noul": 71.73333333333333
      },
      "cc_tiered": 73.63524682706137,
      "cc_tiered_open": 81.04347826086956,
      "cc_tiered_sealed": 66.22701539325317,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 71.73333333333333
     },
     "score": {
      "accuracy": 0.6693333333333333,
      "calibration": 88.7610863978799,
      "cc": 53.323643295776094,
      "cc_by_type": {
       "score": 53.323643295776094
      },
      "cc_tiered": 54.36878620104583,
      "cc_tiered_open": 58.45391141164524,
      "cc_tiered_sealed": 50.28366099044642,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.323643295776094
     }
    },
    "raw_on_A": {
     "calibration": 85.92539384072604,
     "capability": 77.22561866144176,
     "intelligence": 68.52584348215746
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.96641347107422,
    "Emphasis on Accuracy 60:20:20": 66.04732465008613,
    "Emphasis on Cost 20:20:60": 57.80803192071705,
    "Emphasis on Speed 20:60:20": 72.78860583369094,
    "Intelligence only": 67.73785836837352
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bynn Intelligence (Hugging Face)",
   "axes": {
    "calibration": 77.47576560172445,
    "cost": 71.18546346484604,
    "intelligence": 26.810082223231646,
    "speed": 92.88928290241705
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11639344262295086,
      "ece_score": 76.72131147540982,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 76.72131147540982,
      "score_before_time_limit": 76.72131147540982,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.37801608579088475,
      "brier": 0.18461608579088468,
      "ece": 0.12230563002680964,
      "n": 373,
      "score": 75.53887399463808,
      "score_before_time_limit": 75.53887399463808,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8844086021505376,
      "interval90_width": 0.4422043010752688,
      "n": 372,
      "nrps": 0.1265502464157705,
      "score": 80.16711133512545,
      "score_before_time_limit": 80.16711133512545,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.1350537634408602
     }
    },
    "score": 77.47576560172445
   },
   "capability": 52.14292391247805,
   "composite_ci95": {
    "A": [
     9.423742190301025,
     22.25794582937596
    ],
    "B": [
     7.647079700075117,
     19.002863162841397
    ],
    "C": [
     6.544265409931267,
     15.456906825955523
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string Qwen/Qwen3.5-2B-Base (the base named in the repo's LICENSING.md; the module's DeepInfra base-reference table maps it to the Qwen/Qwen3.5-2B listing) priced as Qwen/Qwen3.5-2B USD 0.02/M input, USD 0.1/M output (the non-Base sibling listing; the module pins the DeepInfra/OpenRouter price snapshots of 25 Sep 2026). Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's server: input_tokens = the rendered prompt of the one question (chat template and options included); output_tokens = 0 on every row (the server generates no text: one forward pass and a readout head). The server-reported total of 677,924 input tokens over the 1,485 answered items equals the pre-registered offline count (first-party transcription of the author's prompt renderer and the pinned tokenizer, made before the run), with 0 mismatches item by item and offset 0; the same total applies to both DecisionNode rows. The 15 refused items (HTTP 400) carry no usage and count 0 tokens. Those 15 are the 15 standard very long items, which count for no row's cost on the common 1,485-item cost basis (v1.6.30); the common-basis input tokens are therefore also 677,924, with 0 output tokens. The preregistration (Amendment 2) predicted USD 0.009039 per 1,000 decisions on the 1,500-item basis (0.1399x the cost cap) and USD 0.009130 on the common 1,485-item basis (0.1413x); the measured token total equals the predicted one.  COST ON THE COMMON BASIS: USD 0.009130 per 1,000 decisions on 1,485 items, 0.141 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009038986666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009130289562289562
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "DecisionNode Edge 0.9.2",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.574/h as recorded by the runner (300 GB disk configured). One VM for both rows back to back with verified node shredding after each row and a final shred. The author's server (127.0.0.1:8000) in a --network none --read-only container per row; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before any sealed input (only loopback, no default route, DNS and TCP fail). Measured 24 Oct 2026 (reservation 05:33:57Z, VM created 05:35:01Z, teardown confirmed 05:43:05Z); this row's measurement container 91 s (run cap 112 s), scored loop 49.329 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 395ad44a0408cc796bca562f7ef9b47eb20fab6320c50273969a0ce0efd9134f. Node-shred receipt verified for each row; exact-id VM and volume teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 33.72323159403544,
    "I_sealed": 19.896932852427852,
    "base": 26.810082223231646,
    "excess": 3.9867719712518763,
    "gap": 13.826298741607587,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 47.260855396922054,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 45.265806857502355,
       "judge": 56.49307844429796,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 18.697722567287787,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 35.211116817896475,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.45454545454546,
       "hard": 23.990740740740733,
       "judge": 32.16949152542372,
       "standard": 50.59259259259259
      }
     },
     "sealed|choice": {
      "cc": 40.910318515951026,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 61.88055908513342,
       "hard": 34.94853706487798,
       "judge": 39.17727717923606,
       "standard": 44.94832313857837
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -3.4482758620689613,
       "hard": -9.090909090909093,
       "judge": -7.692307692307687,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 18.78048004133253,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 32.99122807017544,
       "hard": 11.535211267605682,
       "judge": 16.16666666666665,
       "standard": 30.08636363636361
      }
     }
    }
   },
   "jevbench_score": 15.329900484746961,
   "key": "decisionnode-edge-0.9.2",
   "last_measured_on": "2026-10-24",
   "licence": "AGPL-3.0-only for the weights and the decisionnode_edge runtime (LICENSING.md, LICENSE, manifest licence agpl-3.0; commercial licence offered separately by Bynn Intelligence Inc.); base model Qwen/Qwen3.5-2B-Base Apache-2.0 (LICENSE-base); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "bynn-intelligence-com/decisionnode-edge@0823eb7134172e1c8c8daaa7244a6cc77c5e5ac9",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.8275862068965517,
    "decisive": 232,
    "decisive_rate": 0.6219839142091153,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 178,
   "ranked": true,
   "ranks": {
    "A": 178,
    "B": 179,
    "C": 180,
    "capability": 191
   },
   "repo": "https://huggingface.co/bynn-intelligence-com/decisionnode-edge",
   "scores": {
    "A": 15.329900484746961,
    "B": 12.798868903208827,
    "C": 10.645764225518723
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21247871499999746,
    "p50_s_raw": 0.031239357499998732,
    "p95_s_adjusted": 0.24196707719999608,
    "p95_s_raw": 0.04598353859999804
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: bynn-intelligence-com/decisionnode-edge at 0823eb7134172e1c8c8daaa7244a6cc77c5e5ac9 (tag v0.9.2) with the authors' PyPI package decisionnode-edge 0.9.2 (wheel sha256 dfb0b48c1233c32a106d8eee4d6c320dc3454026a1119851787f345b2f5c6317, sdist sha256 2f1da8de0be9edb3a58c3804dc024cede75b53dcea9aa82a42645e1a189ccbd0): 11 Python modules, installed from the hash-pinned wheel with --no-deps --no-index under a network-free image build; the 11 modules equal (git blob oid) runtime/src/decisionnode_edge in both Hub repos at their pins. Only the 10 data files of the pinned revision were mounted read-only at /model (config.json, decisionnode_edge.json, calibration.json, box_calibration.json, tokenizer.json, tokenizer_config.json, preprocessor_config.json, video_preprocessor_config.json, model.safetensors, readout.safetensors); no Python, GGUF or shell file of the Hub repo was fetched. Command: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 USE_HUB_KERNELS=NO DECISIONNODE_EDGE_BACKEND=torch PYTHONDONTWRITEBYTECODE=1 python3 -m decisionnode_edge serve /model --host 127.0.0.1 --port 8000 --backend torch. PyTorch bf16 reference backend, one request at a time, no optional kernels (flash-linear-attention, causal-conv1d, Hub kernels, openvino and mlx absent; checked in the image). Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 used as the CUDA 13.0 / PyTorch base (vLLM 0.30.0 is installed in it but was not used: the author's default --backend auto would select vllm there, so torch was set on the command line and in the environment, and /healthz reported backend torch after the sealed run), derived image sha256:b1f31eab61d184033ae78a92aa778c7faa2c909f8566725b4716d7803535dd47 built offline from three hash-pinned wheels (decisionnode_edge 0.9.2, transformers 5.18.0 sha256 d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9, huggingface_hub 1.33.0 sha256 04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899). Python 3.12.3, torch 2.13.0+cu130, transformers 5.18.0, tokenizers 0.23.2, safetensors 0.8.0, numpy 2.2.6. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: dormant subprocess start of a llama-server child in the llamacpp backend (backends.py:353), reachable only with --backend llamacpp; GGUF build subprocesses (build.py:134,136), a file imported by nothing and run only as python -m decisionnode_edge.build; Hub-capable from_pretrained loaders (model.py:49,92,235) that receive the local hash-checked model folder, under HF_HUB_OFFLINE=1, TRANSFORMERS_OFFLINE=1 and a container without network. No runtime download, trust_remote_code, string eval/exec, pickle, telemetry or request-body logging in the author code. The source and runner were reviewed before the run, with the required conditions enforced. A leftover .cache directory that would have stopped the run before any sealed item was removed and the fix rechecked; the exact device selection change was also reviewed. Weights: AGPL-3.0, public, ungated, 2,213,372,736 bf16 parameters, 10 data files of 4,449,009,719 bytes in total hash-checked file by file (verdict OK); prereg manifest sha256 948477ee75aa7f7f53f65be22babda905c06ce68fa664d3aa91618ba1385d81a. The Hub main branch of the repo is one commit ahead of the pinned tag (7706b508), with zero file differences against the pinned tag commit (added 0, removed 0, changed 0 in our file-by-file comparison); the measurement stays at the pinned commit 0823eb71. FIRST MEASUREMENT of DecisionNode Edge 0.9.2 (our own intake; filed as GitHub issue 231 by the account @bynnit on 23 Oct 2026). No DecisionNode row exists on the live JevBench board (v1.6.42, checked 24 Oct 2026). AUTHOR'S OWN SERVER on our rented hardware, PyTorch bf16 reference backend, one request at a time. NATIVE OUTPUTS: probabilities over the item's options from the model's readout head with the author's calibration applied by the server (choice argmax and probabilities; score level, expectation and probabilities; noul P(true) from the server's truth answer reported as a noul); the server rounds probabilities to 2 decimals; nothing is generated. NAMING NOTE: the package, the PyPI release and the Hub tags are 0.9.2, but the served model name reads decisionnode-edge-0.9.1 (/healthz, /v1/models and every response): the name string comes from decisionnode_edge.json in the model folder. The release owner's triage of 23 Oct 2026 found model.safetensors, readout.safetensors, config.json, tokenizer.json and calibration.json byte-identical across the tags v0.9.1 and v0.9.2; v0.9.2 adds the /v1/systemone server, box answers and documentation. The measured model tensors are therefore the 0.9.1 weights, served by the 0.9.2 runtime. PYTORCH PATH: the author's default backend selection would have used vLLM in this image; we forced the PyTorch reference path, without the optional flash-linear-attention and causal-conv1d kernels. The author's README says each backend agrees with the PyTorch reference on at least 98% of top answers and that the PyTorch path without the kernels gives the same answers about twice as slowly on long inputs; we did not test either statement. The README's timing (9.5 ms at about 500 tokens, vLLM backend, RTX PRO 6000) does not transfer to this PyTorch measurement. AUTHOR EXPOSURE (volunteered by the author in the submission and recorded in the release owner's triage; not verified by us): no JevBench item is in the training data (every training record was checked against the public items by exact text and overlapping word windows, and matches were dropped), but the public items were one of the authors' development suites: they read scores on them while comparing training runs, which is model selection against the public split. No overlap or contamination audit was run by us. LICENCE NOTE: AGPL-3.0 for the weights and the runtime; Bynn Intelligence Inc. also offers a separate commercial licence (LICENSING.md); whether the page needs any licence line is a release-owner decision. NOT ANSWERED, counted wrong: 15 items exceed the author's 16,384-token context limit (HTTP 400, error_type max_tokens_exceeded), exactly the 15 standard very long items (76,309-80,466 tokens by our first-party offline count) at positions in the pre-registered long-item set, as pre-registered by id and position; never truncated. Answered: 1,485 of the 1,500 items sent, HTTP 200 (status counts 200: 1,485, 400: 15); the longest answered item was 1,794 tokens. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 0.111 s over the 1,500 requests (median 0.0312 s, p95 0.0460 s). DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0 (compared on the server's 2-decimal outputs). CONTEXT: author-imposed limit 16,384 tokens; no truncation. Pre-sealed synthetic probes (not benchmark items): a 16,384-token request was accepted (0.76 s), requests of 16,385 and 82,000 tokens were refused with HTTP 400 and the server stayed healthy. The vendor's reported figures are not independently verified by us.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 68.8888888888889,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 39.99999999999999
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 68.8888888888889
      },
      "calibration_gen": {
       "accuracy": 0.43564356435643564,
       "cc": 12.57659508660304,
       "cc_by_type": {
        "choice": 45.91728525980912,
        "noul": -12.0,
        "score": 3.8124999999999964
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 12.57659508660304
      },
      "composite_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 14.499339865193514,
       "cc_by_type": {
        "choice": 28.45528455284553,
        "noul": -11.111111111111116,
        "score": 26.15384615384613
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 14.499339865193514
      },
      "fairness": {
       "accuracy": 0.6666666666666666,
       "cc": 58.666666666666664,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 46.66666666666666,
        "score": 29.333333333333357
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 58.666666666666664
      },
      "jaggedness": {
       "accuracy": 0.3,
       "cc": -17.725665313164612,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -67.56756756756756,
        "score": -0.4313725490196374
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6773162939297125,
       "cc": 51.702956719851535,
       "cc_by_type": {
        "choice": 53.24469490182266,
        "noul": 54.639175257731964,
        "score": 47.224999999999994
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 51.702956719851535
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -2.345442644411713,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25.0,
        "score": -7.8095238095237995
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5781818181818181,
       "cc": 29.856090351477416,
       "cc_by_type": {
        "choice": 55.0048907727421,
        "noul": 0.0,
        "score": 34.56338028169014
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 29.856090351477416
      },
      "negation_pair_gen": {
       "accuracy": 0.45348837209302323,
       "cc": 1.1666666666666685,
       "cc_by_type": {
        "choice": 29.999999999999996,
        "noul": -19.999999999999996,
        "score": -6.499999999999995
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 1.1666666666666685
      },
      "option_order": {
       "accuracy": 0.4583333333333333,
       "cc": 30.758988015978694,
       "cc_by_type": {
        "choice": 30.758988015978694
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 30.758988015978694
      },
      "perturb_pair": {
       "accuracy": 0.703125,
       "cc": 51.24339724339725,
       "cc_by_type": {
        "choice": 60.96096096096097,
        "noul": 39.99999999999999,
        "score": 52.76923076923077
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 51.24339724339725
      },
      "usecase": {
       "accuracy": 0.45454545454545453,
       "cc": 10.922483660130728,
       "cc_by_type": {
        "choice": 43.627450980392176,
        "noul": -37.5,
        "score": 26.640000000000008
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 10.922483660130728
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -7.428571428571431,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 0.0,
        "score": -8.000000000000007
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 34.25555555555554,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 36.099999999999966
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 34.25555555555554
      },
      "de": {
       "accuracy": 0.75,
       "cc": 69.27369501090163,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 100.0,
        "score": 55.60869565217392
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 69.27369501090163
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 16.178150217256363,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 99.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.178150217256363
      },
      "en": {
       "accuracy": 0.49869678540399653,
       "cc": 19.641346933406236,
       "cc_by_type": {
        "choice": 39.81035135036182,
        "noul": -1.4388489208633115,
        "score": 20.552538370720196
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 19.641346933406236
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 47.07481687617951,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100.0,
        "score": -29.411764705882337
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 47.07481687617951
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 8.586520947176682,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 24.666666666666657
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 8.586520947176682
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 44.67801857585139,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 55.210526315789465
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 44.67801857585139
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 11.746741154562386,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0.0,
        "score": -48.0
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 11.746741154562386
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 41.21834984968265,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -48.285714285714306
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 41.21834984968265
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 32.59182772923837,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 20.66666666666668
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 32.59182772923837
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 13.632275132275131,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 0.0,
        "score": 29.78571428571428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 13.632275132275131
      },
      "mixed": {
       "accuracy": 0.7142857142857143,
       "cc": 63.166666666666664,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 33.33333333333333,
        "score": 89.5
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 63.166666666666664
      },
      "nl": {
       "accuracy": 0.4,
       "cc": 13.123520923520921,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -33.333333333333336,
        "score": 76.6
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 13.123520923520921
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 48.154267830605555,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 19.697674418604638
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 48.154267830605555
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 65.99718034989006,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 40.41935483870967
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.99718034989006
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -10.209219858156024,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100.0,
        "score": 71.50000000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 7.427536231884059,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0.0,
        "score": -12.5
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 7.427536231884059
      },
      "uk": {
       "accuracy": 1.0,
       "cc": 96.16666666666667,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 88.5
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 96.16666666666667
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 13.080979284369107,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -2.000000000000024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 13.080979284369107
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5613333333333334,
       "cc": 43.339903174572804,
       "cc_by_type": {
        "choice": 43.339903174572804
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 43.339903174572804
      },
      "noul": {
       "accuracy": 0.512,
       "cc": 2.400000000000002,
       "cc_by_type": {
        "noul": 2.400000000000002
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 2.400000000000002
      },
      "score": {
       "accuracy": 0.448,
       "cc": 22.890794223826717,
       "cc_by_type": {
        "score": 22.890794223826717
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 22.890794223826717
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -10.70588235294118,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -100.0,
        "score": 31.88235294117647
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 19.284192458105505,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -11.111111111111116,
        "score": 6.23076923076924
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 19.284192458105505
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 78.24661246612466,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100.0,
        "score": 51.00000000000002
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 78.24661246612466
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -24.921052631578956,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": 30.499999999999982
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 22.287439613526573,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": -8.499999999999996
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.287439613526573
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -142.04297693920336,
       "cc_by_type": {
        "choice": -0.62893081761006,
        "noul": -100.0,
        "score": -325.5
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 8.24678589384471,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": 35.38461538461537
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 8.24678589384471
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -29.930162096652026,
       "cc_by_type": {
        "choice": -4.529616724738673,
        "noul": -100.0,
        "score": 14.739130434782599
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 11.18518518518519,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 56.05555555555557
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.18518518518519
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -0.7767395583241322,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 8.090909090909115
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.44,
       "cc": 0.15378592122778892,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -55.55555555555556,
        "score": -20.727272727272705
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.15378592122778892
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -3.2831541218638036,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": -12.00000000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 21.687388924433062,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 19.61111111111109
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 21.687388924433062
      },
      "model_routing": {
       "accuracy": 0.6912442396313364,
       "cc": 51.99623466810966,
       "cc_by_type": {
        "choice": 61.42857142857144,
        "noul": 57.57575757575757,
        "score": 36.98437499999998
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 51.99623466810966
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 31.68412931391366,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 0.0,
        "score": 41.11764705882352
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 31.68412931391366
      },
      "recruiting": {
       "accuracy": 0.5,
       "cc": 15.080160940885184,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 12.000000000000032
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 15.080160940885184
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -13.665217811262439,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -100.0,
        "score": 18.117647058823515
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 20.942995169082124,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 54.133333333333326
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 20.942995169082124
      },
      "search_retrieval": {
       "accuracy": 0.7083333333333334,
       "cc": 47.76512455516015,
       "cc_by_type": {
        "choice": 57.295373665480426,
        "noul": 0.0,
        "score": 86.00000000000001
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 47.76512455516015
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.59338177135288,
      80.56583679326516
     ],
     "I": [
      22.166108712128455,
      31.06134159919504
     ],
     "capability": [
      47.643755360051685,
      55.018609381039205
     ],
     "composite": {
      "A": [
       9.423742190301025,
       22.25794582937596
      ],
      "B": [
       7.647079700075117,
       19.002863162841397
      ],
      "C": [
       6.544265409931267,
       15.456906825955523
      ]
     },
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.47576560172445,
     "capability": 52.14292391247805,
     "intelligence": 26.810082223231646
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5613333333333334,
      "calibration": 76.72131147540982,
      "cc": 43.339903174572804,
      "cc_by_type": {
       "choice": 43.339903174572804
      },
      "cc_tiered": 44.08558695643654,
      "cc_tiered_open": 47.260855396922054,
      "cc_tiered_sealed": 40.910318515951026,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 43.339903174572804
     },
     "noul": {
      "accuracy": 0.512,
      "calibration": 75.53887399463808,
      "cc": 2.400000000000002,
      "cc_by_type": {
       "noul": 2.400000000000002
      },
      "cc_tiered": 9.348861283643894,
      "cc_tiered_open": 18.697722567287787,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 2.400000000000002
     },
     "score": {
      "accuracy": 0.448,
      "calibration": 80.16711133512545,
      "cc": 22.890794223826717,
      "cc_by_type": {
       "score": 22.890794223826717
      },
      "cc_tiered": 26.995798429614503,
      "cc_tiered_open": 35.211116817896475,
      "cc_tiered_sealed": 18.78048004133253,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.890794223826717
     }
    },
    "raw_on_A": {
     "calibration": 79.09742907123326,
     "capability": 53.09845110814928,
     "intelligence": 27.099473145065282
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 13.886634650410828,
    "Emphasis on Accuracy 60:20:20": 10.515288194574055,
    "Emphasis on Cost 20:20:60": 15.936009934439225,
    "Emphasis on Speed 20:60:20": 17.186716574951603,
    "Intelligence only": 7.70822581673003
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bynn Intelligence (Hugging Face)",
   "axes": {
    "calibration": 77.66777221653875,
    "cost": 80.21636333476548,
    "intelligence": 17.55610652928715,
    "speed": 92.90461510807665
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10416393442622954,
      "ece_score": 79.1672131147541,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 79.1672131147541,
      "score_before_time_limit": 79.1672131147541,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.5308310991957105,
      "brier": 0.21568900804289548,
      "ece": 0.10321715817694371,
      "n": 373,
      "score": 79.35656836461126,
      "score_before_time_limit": 79.35656836461126,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8844086021505376,
      "interval90_width": 0.5495071684587812,
      "n": 372,
      "nrps": 0.15062435035842298,
      "score": 74.4795351702509,
      "score_before_time_limit": 74.4795351702509,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.17989247311827955
     }
    },
    "score": 77.66777221653875
   },
   "capability": 47.61193937291295,
   "composite_ci95": {
    "A": [
     2.621380879069712,
     7.988511226445658
    ],
    "B": [
     1.9631410062105725,
     6.353179629886551
    ],
    "C": [
     1.8204033882428559,
     5.5475772405872625
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string Qwen/Qwen3.5-0.8B-Base (the base named in the repo's LICENSING.md; the module's DeepInfra base-reference table maps it to the Qwen/Qwen3.5-0.8B listing) priced as Qwen/Qwen3.5-0.8B USD 0.01/M input, USD 0.05/M output (the non-Base sibling listing; the module pins the DeepInfra/OpenRouter price snapshots of 25 Sep 2026). Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's server: input_tokens = the rendered prompt of the one question (chat template and options included); output_tokens = 0 on every row (the server generates no text: one forward pass and a readout head). The server-reported total of 677,924 input tokens over the 1,485 answered items equals the pre-registered offline count (first-party transcription of the author's prompt renderer and the pinned tokenizer, made before the run), with 0 mismatches item by item and offset 0; the same total applies to both DecisionNode rows. The 15 refused items (HTTP 400) carry no usage and count 0 tokens. Those 15 are the 15 standard very long items, which count for no row's cost on the common 1,485-item cost basis (v1.6.30); the common-basis input tokens are therefore also 677,924, with 0 output tokens. The preregistration (Amendment 2) predicted USD 0.004519 per 1,000 decisions on the 1,500-item basis (0.0700x the cost cap) and USD 0.004565 on the common 1,485-item basis (0.0707x); the measured token total equals the predicted one.  COST ON THE COMMON BASIS: USD 0.004565 per 1,000 decisions on 1,485 items, 0.071 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0045194933333333335
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004565144781144781
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "DecisionNode Edge Mini 0.9.2",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.574/h as recorded by the runner (300 GB disk configured). One VM for both rows back to back with verified node shredding after each row and a final shred. The author's server (127.0.0.1:8000) in a --network none --read-only container per row; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before any sealed input (only loopback, no default route, DNS and TCP fail). Measured 24 Oct 2026 (reservation 05:33:57Z, VM created 05:35:01Z, teardown confirmed 05:43:05Z); this row's measurement container 91 s (run cap 109 s), scored loop 49.297 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 46b320c66dc0821d94fcd4a60a58a837d06f331357002b244e7e6c7fa4176cb3. Node-shred receipt verified for each row; exact-id VM and volume teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 20.5876783664135,
    "I_sealed": 14.524534692160799,
    "base": 17.55610652928715,
    "excess": -3.776383096103011,
    "gap": 6.0631436742527,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 36.313916797579814,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 32.05410506448569,
       "judge": 32.76203032300594,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 25.44911830166068,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.272727272727266,
       "hard": 15.601851851851867,
       "judge": 30.81355932203388,
       "standard": 32.1851851851852
      }
     },
     "sealed|choice": {
      "cc": 34.46999359481248,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 54.2566709021601,
       "hard": 28.703596623106264,
       "judge": 33.00685602350637,
       "standard": 38.30415524151024
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -24.13793103448276,
       "hard": -34.54545454545455,
       "judge": -32.30769230769231,
       "standard": -22.388059701492537
      }
     },
     "sealed|score": {
      "cc": 9.103610481669918,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 8.447368421052648,
       "hard": 4.9887323943662025,
       "judge": 11.671874999999975,
       "standard": 13.809090909090893
      }
     }
    }
   },
   "jevbench_score": 5.29891802613964,
   "key": "decisionnode-edge-mini-0.9.2",
   "last_measured_on": "2026-10-24",
   "licence": "AGPL-3.0-only for the weights and the decisionnode_edge runtime (LICENSING.md, LICENSE, manifest licence agpl-3.0; commercial licence offered separately by Bynn Intelligence Inc.); base model Qwen/Qwen3.5-0.8B-Base Apache-2.0 (LICENSE-base); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "bynn-intelligence-com/decisionnode-edge-mini@b1ca52601cd95ef543bbe2bd0cebae2ed3f3b3fa",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.7885714285714286,
    "decisive": 175,
    "decisive_rate": 0.4691689008042895,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 207,
   "ranked": true,
   "ranks": {
    "A": 207,
    "B": 207,
    "C": 207,
    "capability": 204
   },
   "repo": "https://huggingface.co/bynn-intelligence-com/decisionnode-edge-mini",
   "scores": {
    "A": 5.29891802613964,
    "B": 4.108850512138984,
    "C": 3.679804184819195
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21277779499998814,
    "p50_s_raw": 0.03138889749999407,
    "p95_s_adjusted": 0.2407754400000101,
    "p95_s_raw": 0.04538772000000506
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: bynn-intelligence-com/decisionnode-edge-mini at b1ca52601cd95ef543bbe2bd0cebae2ed3f3b3fa (tag v0.9.2) with the authors' PyPI package decisionnode-edge 0.9.2 (wheel sha256 dfb0b48c1233c32a106d8eee4d6c320dc3454026a1119851787f345b2f5c6317, sdist sha256 2f1da8de0be9edb3a58c3804dc024cede75b53dcea9aa82a42645e1a189ccbd0): 11 Python modules, installed from the hash-pinned wheel with --no-deps --no-index under a network-free image build; the 11 modules equal (git blob oid) runtime/src/decisionnode_edge in both Hub repos at their pins. Only the 10 data files of the pinned revision were mounted read-only at /model (config.json, decisionnode_edge.json, calibration.json, box_calibration.json, tokenizer.json, tokenizer_config.json, preprocessor_config.json, video_preprocessor_config.json, model.safetensors, readout.safetensors); no Python, GGUF or shell file of the Hub repo was fetched. Command: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 USE_HUB_KERNELS=NO DECISIONNODE_EDGE_BACKEND=torch PYTHONDONTWRITEBYTECODE=1 python3 -m decisionnode_edge serve /model --host 127.0.0.1 --port 8000 --backend torch. PyTorch bf16 reference backend, one request at a time, no optional kernels (flash-linear-attention, causal-conv1d, Hub kernels, openvino and mlx absent; checked in the image). Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 used as the CUDA 13.0 / PyTorch base (vLLM 0.30.0 is installed in it but was not used: the author's default --backend auto would select vllm there, so torch was set on the command line and in the environment, and /healthz reported backend torch after the sealed run), derived image sha256:b1f31eab61d184033ae78a92aa778c7faa2c909f8566725b4716d7803535dd47 built offline from three hash-pinned wheels (decisionnode_edge 0.9.2, transformers 5.18.0 sha256 d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9, huggingface_hub 1.33.0 sha256 04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899). Python 3.12.3, torch 2.13.0+cu130, transformers 5.18.0, tokenizers 0.23.2, safetensors 0.8.0, numpy 2.2.6. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: dormant subprocess start of a llama-server child in the llamacpp backend (backends.py:353), reachable only with --backend llamacpp; GGUF build subprocesses (build.py:134,136), a file imported by nothing and run only as python -m decisionnode_edge.build; Hub-capable from_pretrained loaders (model.py:49,92,235) that receive the local hash-checked model folder, under HF_HUB_OFFLINE=1, TRANSFORMERS_OFFLINE=1 and a container without network. No runtime download, trust_remote_code, string eval/exec, pickle, telemetry or request-body logging in the author code. The source and runner were reviewed before the run, with the required conditions enforced. A leftover .cache directory that would have stopped the run before any sealed item was removed and the fix rechecked; the exact device selection change was also reviewed. Weights: AGPL-3.0, public, ungated, 853,051,456 bf16 parameters, 10 data files of 1,727,281,260 bytes in total hash-checked file by file (verdict OK); prereg manifest sha256 4c7373fe2e2f94ef2f0df038708580dd2acbdae840f595142155c2cae1e27abb. The Hub main branch of the repo is one commit ahead of the pinned tag (0e77dfdc), with zero file differences against the pinned tag commit (added 0, removed 0, changed 0 in our file-by-file comparison); the measurement stays at the pinned commit b1ca5260. FIRST MEASUREMENT of DecisionNode Edge Mini 0.9.2 (our own intake; filed as GitHub issue 231 by the account @bynnit on 23 Oct 2026). No DecisionNode row exists on the live JevBench board (v1.6.42, checked 24 Oct 2026). AUTHOR'S OWN SERVER on our rented hardware, PyTorch bf16 reference backend, one request at a time. NATIVE OUTPUTS: probabilities over the item's options from the model's readout head with the author's calibration applied by the server (choice argmax and probabilities; score level, expectation and probabilities; noul P(true) from the server's truth answer reported as a noul); the server rounds probabilities to 2 decimals; nothing is generated. NAMING NOTE: the package, the PyPI release and the Hub tags are 0.9.2, but the served model name reads decisionnode-edge-mini-0.9.1 (/healthz, /v1/models and every response): the name string comes from decisionnode_edge.json in the model folder. The release owner's triage of 23 Oct 2026 found model.safetensors, readout.safetensors, config.json, tokenizer.json and calibration.json byte-identical across the tags v0.9.1 and v0.9.2; v0.9.2 adds the /v1/systemone server, box answers and documentation. The measured model tensors are therefore the 0.9.1 weights, served by the 0.9.2 runtime. PYTORCH PATH: the author's default backend selection would have used vLLM in this image; we forced the PyTorch reference path, without the optional flash-linear-attention and causal-conv1d kernels. The author's README says each backend agrees with the PyTorch reference on at least 98% of top answers and that the PyTorch path without the kernels gives the same answers about twice as slowly on long inputs; we did not test either statement. The README's timing (7.3 ms at about 500 tokens, vLLM backend, RTX PRO 6000) does not transfer to this PyTorch measurement. AUTHOR EXPOSURE (volunteered by the author in the submission and recorded in the release owner's triage; not verified by us): no JevBench item is in the training data (every training record was checked against the public items by exact text and overlapping word windows, and matches were dropped), but the public items were one of the authors' development suites: they read scores on them while comparing training runs, which is model selection against the public split. No overlap or contamination audit was run by us. LICENCE NOTE: AGPL-3.0 for the weights and the runtime; Bynn Intelligence Inc. also offers a separate commercial licence (LICENSING.md); whether the page needs any licence line is a release-owner decision. NOT ANSWERED, counted wrong: 15 items exceed the author's 16,384-token context limit (HTTP 400, error_type max_tokens_exceeded), exactly the 15 standard very long items (76,309-80,466 tokens by our first-party offline count) at positions in the pre-registered long-item set, as pre-registered by id and position; never truncated. Answered: 1,485 of the 1,500 items sent, HTTP 200 (status counts 200: 1,485, 400: 15); the longest answered item was 1,794 tokens. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 0.108 s over the 1,500 requests (median 0.0314 s, p95 0.0454 s). DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0 (compared on the server's 2-decimal outputs). CONTEXT: author-imposed limit 16,384 tokens; no truncation. Pre-sealed synthetic probes (not benchmark items): a 16,384-token request was accepted (0.64 s), requests of 16,385 and 82,000 tokens were refused with HTTP 400 and the server stayed healthy. The vendor's reported figures are not independently verified by us.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 59.555555555555564,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 12.00000000000001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 59.555555555555564
      },
      "calibration_gen": {
       "accuracy": 0.3564356435643564,
       "cc": -6.900660569105679,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -52.0,
        "score": 4.468750000000032
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.40594059405940597,
       "cc": -3.3694670280036214,
       "cc_by_type": {
        "choice": 15.447154471544714,
        "noul": -55.55555555555556,
        "score": 29.999999999999982
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 19.568783068783066,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": -19.999999999999996,
        "score": 7.2777777777777715
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 19.568783068783066
      },
      "jaggedness": {
       "accuracy": 0.3,
       "cc": -21.841324368018586,
       "cc_by_type": {
        "choice": 16.265640038498567,
        "noul": -62.16216216216216,
        "score": -19.627450980392158
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.5143769968051118,
       "cc": 24.437463548299828,
       "cc_by_type": {
        "choice": 40.56529012943559,
        "noul": 9.278350515463906,
        "score": 23.468749999999982
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 24.437463548299828
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -12.437407952871878,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50.0,
        "score": -0.714285714285734
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.49818181818181817,
       "cc": 17.71242443369172,
       "cc_by_type": {
        "choice": 45.2233452885556,
        "noul": -13.888888888888884,
        "score": 21.80281690140844
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 17.71242443369172
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -13.444444444444448,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -40.0,
        "score": -17.000000000000014
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.4166666666666667,
       "cc": 25.43275632490014,
       "cc_by_type": {
        "choice": 25.43275632490014
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 25.43275632490014
      },
      "perturb_pair": {
       "accuracy": 0.640625,
       "cc": 37.113190113190115,
       "cc_by_type": {
        "choice": 54.95495495495495,
        "noul": 0.0,
        "score": 56.38461538461539
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 37.113190113190115
      },
      "usecase": {
       "accuracy": 0.42272727272727273,
       "cc": 2.1034771241830064,
       "cc_by_type": {
        "choice": 46.07843137254903,
        "noul": -62.5,
        "score": 22.731999999999985
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 2.1034771241830064
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": 3.0134310134310094,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0.0,
        "score": -14.769230769230779
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 3.0134310134310094
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 23.664004096262158,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 45.800000000000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 23.664004096262158
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 46.290881108118505,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 50.0,
        "score": 28.695652173913043
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 46.290881108118505
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.098075729360644,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 100.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.098075729360644
      },
      "en": {
       "accuracy": 0.42224152910512597,
       "cc": 3.098790707980498,
       "cc_by_type": {
        "choice": 32.121892581247444,
        "noul": -33.09352517985612,
        "score": 10.268004722550172
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 3.098790707980498
      },
      "es": {
       "accuracy": 0.5806451612903226,
       "cc": 37.726001983175,
       "cc_by_type": {
        "choice": 51.06035889070147,
        "noul": 75.0,
        "score": -12.882352941176457
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 37.726001983175
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -13.009107468123856,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": 59.33333333333335
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 30.139318885448915,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0.0,
        "score": 53.94736842105263
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 30.139318885448915
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": -1.798184357541892,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25.0,
        "score": -46.87499999999998
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 41.12311175444456,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": -48.57142857142858
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 41.12311175444456
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 18.537251787132636,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0.0,
        "score": 12.799999999999978
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 18.537251787132636
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -10.219576719576722,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 13.78571428571428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.7142857142857143,
       "cc": 60.166666666666664,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 33.33333333333333,
        "score": 97.16666666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 60.166666666666664
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -2.502453102453105,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -48.20000000000002
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 45.10349476916414,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 39.99999999999999,
        "score": 18.162790697674414
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.10349476916414
      },
      "pt": {
       "accuracy": 0.5714285714285714,
       "cc": 30.429271136287316,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 19.999999999999996,
        "score": 24.322580645161295
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 30.429271136287316
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -26.542553191489365,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100.0,
        "score": 22.499999999999986
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.23076923076923078,
       "cc": -23.398550724637683,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0.0,
        "score": -61.5
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 11.833333333333334,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 35.499999999999986
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 11.833333333333334
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -51.34274952919021,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -33.333333333333336,
        "score": -119.0
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5013333333333333,
       "cc": 35.59004190665723,
       "cc_by_type": {
        "choice": 35.59004190665723
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 35.59004190665723
      },
      "noul": {
       "accuracy": 0.368,
       "cc": -26.400000000000002,
       "cc_by_type": {
        "noul": -26.400000000000002
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.3973333333333333,
       "cc": 12.228339350180606,
       "cc_by_type": {
        "score": 12.228339350180606
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 12.228339350180606
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.941176470588242,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -100.0,
        "score": 24.176470588235276
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -11.905620144750577,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -77.77777777777779,
        "score": -2.0384615384615223
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 55.40650406504064,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 64.99999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.40650406504064
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.70711500974658,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -16.249999999999986
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": 15.20652173913043,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": 33.33333333333333,
        "score": -15.250000000000007
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 15.20652173913043
      },
      "feature_extraction": {
       "accuracy": 0.21052631578947367,
       "cc": -88.01677148846962,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100.0,
        "score": -176.00000000000003
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": 1.9702650290885515,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": 26.35897435897434
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 1.9702650290885515
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -19.860071706307128,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100.0,
        "score": 24.043478260869545
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -15.777777777777771,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100.0,
        "score": 56.83333333333336
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.09090909090909091,
       "cc": -32.09457530335474,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -100.0,
        "score": 6.500000000000017
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": -1.0689844178216248,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -77.77777777777779,
        "score": 9.45454545454546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -15.502688172043015,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100.0,
        "score": -14.250000000000007
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.4444444444444444,
       "cc": -5.779395038032277,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 5.694444444444413
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.5898617511520737,
       "cc": 32.08239763708514,
       "cc_by_type": {
        "choice": 60.259740259740276,
        "noul": 15.15151515151516,
        "score": 20.83593749999998
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 32.08239763708514
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": -7.629596176282415,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 23.176470588235286
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.5,
       "cc": 38.33391662500298,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 33.33333333333333,
        "score": 31.807692307692324
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 38.33391662500298
      },
      "risk_assessment": {
       "accuracy": 0.2222222222222222,
       "cc": -23.473679126823143,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -100.0,
        "score": 3.4705882352941253
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": 12.323188405797097,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 42.76666666666665
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 12.323188405797097
      },
      "search_retrieval": {
       "accuracy": 0.5833333333333334,
       "cc": 34.23003683586189,
       "cc_by_type": {
        "choice": 57.295373665480426,
        "noul": 0.0,
        "score": 45.39473684210525
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 34.23003683586189
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.59220018450009,
      79.71111359708684
     ],
     "I": [
      13.478279881705168,
      20.593532034549767
     ],
     "capability": [
      43.4654134135744,
      49.54844595245572
     ],
     "composite": {
      "A": [
       2.621380879069712,
       7.988511226445658
      ],
      "B": [
       1.9631410062105725,
       6.353179629886551
      ],
      "C": [
       1.8204033882428559,
       5.5475772405872625
      ]
     },
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.66777221653875,
     "capability": 47.61193937291295,
     "intelligence": 17.55610652928715
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5013333333333333,
      "calibration": 79.1672131147541,
      "cc": 35.59004190665723,
      "cc_by_type": {
       "choice": 35.59004190665723
      },
      "cc_tiered": 35.39195519619615,
      "cc_tiered_open": 36.313916797579814,
      "cc_tiered_sealed": 34.46999359481248,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 35.59004190665723
     },
     "noul": {
      "accuracy": 0.368,
      "calibration": 79.35656836461126,
      "cc": -26.400000000000002,
      "cc_by_type": {
       "noul": -26.400000000000002
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.3973333333333333,
      "calibration": 74.4795351702509,
      "cc": 12.228339350180606,
      "cc_by_type": {
       "score": 12.228339350180606
      },
      "cc_tiered": 17.276364391665297,
      "cc_tiered_open": 25.44911830166068,
      "cc_tiered_sealed": 9.103610481669918,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 12.228339350180606
     }
    },
    "raw_on_A": {
     "calibration": 76.68973300664038,
     "capability": 47.40647606143968,
     "intelligence": 18.123219116238985
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 4.612284558808343,
    "Emphasis on Accuracy 60:20:20": 3.175681055647796,
    "Emphasis on Cost 20:20:60": 5.863938580688909,
    "Emphasis on Speed 20:60:20": 6.060240426091981,
    "Intelligence only": 2.164435326956283
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "baselabs (Base Labs at Baseten; baselabs on Hugging Face)",
   "axes": {
    "calibration": 84.65640909439074,
    "cost": 48.3964659766053,
    "intelligence": 55.83724375498836,
    "speed": 89.7330689398618
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10618694754098373,
      "ece_score": 78.76261049180326,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 78.76261049180326,
      "score_before_time_limit": 78.76261049180326,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.21715817694369977,
      "brier": 0.11100565751402397,
      "ece": 0.05619157372654143,
      "n": 373,
      "score": 88.76168525469171,
      "score_before_time_limit": 88.76168525469171,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9354838709677419,
      "interval90_width": 0.3230286738351253,
      "n": 372,
      "nrps": 0.07802430475032547,
      "score": 86.44493153667727,
      "score_before_time_limit": 86.44493153667727,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09653853225806448
     }
    },
    "score": 84.65640909439074
   },
   "capability": 70.24682642468954,
   "composite_ci95": {
    "A": [
     58.43477071734329,
     62.13098693849953
    ],
    "B": [
     55.76855695551602,
     60.812107579242856
    ],
    "C": [
     40.92376735438138,
     61.95925697779976
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the rates for this base string are identical in the earlier module 81e3578f): base string Qwen/Qwen3.5-9B (the card's base_model) priced as qwen/qwen3.5-9b USD 0.10/M input, USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of OneJev's server (create_app, behind our launcher; input_tokens = prefix plus all suffix tokens, incl. system prompt and chat template; output_tokens = number of questions = 1 per answered single-question item, by design, nothing generated). Our offline count through a first-party replica of the engine's pre-GPU path (bl_render.py) and the pinned tokenizer predicted the server's numbers EXACTLY, item by item: 777,323 input tokens and 1,485 output tokens over the 1,485 answered items, and 15 HTTP 422 refusals at the pre-registered ids and positions. COST ON THE COMMON BASIS: USD 0.052495 per 1,000 decisions (USD 0.051970 over all 1,500 items) is 0.813x the official cap USD 0.064595, inside the cap. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them); their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.051970033333333325
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.052494983164983154
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "baselabs elephant-1 (9B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. The launcher and engine run in a docker container with network none (cut proven from inside before any item hash or model load: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only model and input, empty credential variables, HF offline flags, one request at a time, timed over loopback on the VM. Peak CUDA memory allocated after the scored run 22,898 MiB of 97,887 MiB. Measured 22 Oct 2026 (row 10:27-10:32Z, scored run 198 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked on the node and inside the container). This is an elephant-only retry on a fresh VM: the first attempt's elephant row, planned on the mammoth-1 VM, failed before any sealed byte was read (the prepared stack image had been pruned by the mammoth cleanup). On this VM the input was shredded from the node and its absence verified, and the VM was removed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 61.08649662617839,
    "I_sealed": 50.58799088379833,
    "base": 55.83724375498836,
    "excess": 0.6589789720243502,
    "gap": 10.498505742380061,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 73.67205398217818,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 66.02705253224286,
       "judge": 80.22412656558998,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 51.134575569358184,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 65.21739130434783,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 58.45286032699879,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.72297878787879,
       "hard": 60.82901203703703,
       "judge": 57.210411864406765,
       "standard": 49.92917037037038
      }
     },
     "sealed|choice": {
      "cc": 67.82247430702198,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 82.84625158831004,
       "hard": 59.407887128483864,
       "judge": 66.50342801175319,
       "standard": 79.11832946635731
      }
     },
     "sealed|noul": {
      "cc": 37.396837850775064,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 27.27272727272727,
       "judge": 35.384615384615394,
       "standard": 55.223880597014926
      }
     },
     "sealed|score": {
      "cc": 46.544660493597945,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.84345,
       "hard": 39.7783152112676,
       "judge": 45.315041666666666,
       "standard": 49.77238454545453
      }
     }
    }
   },
   "jevbench_score": 60.9075475517525,
   "key": "elephant-1",
   "last_measured_on": "2026-10-22",
   "licence": "MIT (model card metadata since 21 Oct 2026; no LICENSE file; base Qwen/Qwen3.5-9B is Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "baselabs/elephant-1@c54a55c2d362a7e30ff30814d28e39054e954d58",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.910958904109589,
    "decisive": 292,
    "decisive_rate": 0.7828418230563002,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 64,
   "ranked": true,
   "ranks": {
    "A": 64,
    "B": 60,
    "C": 53,
    "capability": 90
   },
   "repo": "https://huggingface.co/baselabs/elephant-1",
   "scores": {
    "A": 60.9075475517525,
    "B": 58.96995444199752,
    "C": 52.749282544598
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3007341589999669,
    "p50_s_raw": 0.07536707949998345,
    "p95_s_adjusted": 0.35359844430012116,
    "p95_s_raw": 0.10179922215006058
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The authors' own serving runtime: OneJev qev package at 81ce62f1597c91e46767d4d02d6ac2e18534fe94 plus the authors' serve.patch (sha256 f234c125f1d38d82104b9e1f24e75bde51e4401928acfba2fbf5c59ef2fb46b8; 15 qev/ files, manifest sha256 818ca0fc1d26ec6a2369aac210fb1de41d908a9e52b45e25bc6150714f046501), vendored as byte-pinned files (re-verified inside the smoke container: 15 files OK), no git clone, no pip -e, setup.sh never run. The authors' code bundle (engine.py, setup.sh, constraints.txt, serve.patch, readout.json) is byte-identical to the one of baselabs/mammoth-1; only config.json, generation_config.json and the weights differ. It is run through OUR first-party launcher baselabs_launcher.py (sha256 7a0896373a6677034909ffa56f17558cbd9ceaf389a9dafdea59cfc638ce26ed), which builds the authors' qev.engine.DecisionEngine (bfloat16 + fp32 LM head, block-bidirectional attention on the full-attention layers, slot_lm readout, 256 slots, debias 1, no calibration file so T=1, auto fork mode which ran as 'batched', author limits 32,768 / 65,536 tokens) and hands it to OneJev's own create_app on 127.0.0.1:8000. `qev serve` was NOT used: it routes these checkpoints (config has a vision_config) to MMDecisionEngine, which never attaches the block-bidirectional mask; the DecisionEngine is what the authors' own Decision Index runs use. The launcher adds only the authors' own 32BitIndexMath sequential-fork fallback and one read-only /_launcher/info route; no prompt, truncation, calibration, answer or usage change. After the scored run the engine reported noncausal attention with the mask installed and every counted mask build (3,000 of 3,000) carried block ids; the counters show that the mask was built with block ids, not numerical parity with the Decision Index runs or with a causal-mask comparison (neither was run). Code gate: cleared by the benchmark owner (22 Oct 2026 08:42Z); a later addendum covering the attention-mask monkeypatch, the Hub-kernel path and the setup.sh extras was disclosed to the owner afterwards and no fresh confirmation was claimed. DISCLOSED CODE PATTERNS (download/client/kernel paths are inactive; the attention-mask monkeypatch executes offline as described below): setup.sh (git clone, pip install -e, a CUDA extension build that can download a wheel), qev/client.py (urllib HTTP client class, unused), qev/engine.py and mm_engine.py Hub loaders (a local verified snapshot is passed, offline flags set), readout.py torch.load of a head file (checkpoints say slot_lm, no head file), constraints.txt listing wandb and an OpenTelemetry exporter stack (not installed), an in-process monkeypatch of transformers' create_causal_mask to add block ids (readout.py:104-124, runs inside the network-none container), and transformers' optional Hub-kernel path (inactive: the kernels package is not installed). A static source review and two independent runner reviews preceded the run (their one must was closed and re-checked before launch); the later runner changes for this elephant-only retry (keep the stack image) were checked by us (diff and dry-run), not by a further review round. Image: our own build on the pod (network on, before any sealed byte) from pytorch/pytorch:2.8.0-cuda12.8-cudnn9-devel@sha256:a7103283ea7113e10ae5d014bd2342acebda0bc53164b2f7b1dd6eb7a766bdb6 plus 50 hash-pinned wheels (--require-hashes, lock sha256 bf9fa61b272ce83fe5a2ac4c6db219484f929083d46efe0f0a5252ab969c31da, installed with --no-index --no-deps); torch 2.8.0+cu128, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2; causal-conv1d is not installed (transformers uses its torch fallback); stack image sha256:76a3333125ebdae54475f49994d3e01ea758a0f82c1f85c0ccdf0ebf03e8dbd5, row image sha256:d4f70ee15d359d0c64a093013bf9bab19f4d4e877469c89142df68f39dc00a29. Weights: baselabs/elephant-1 at c54a55c2d362a7e30ff30814d28e39054e954d58 (licence flip commit 5494c13e; weights commit 14537f08, unchanged since), the 8 serving files (18,839,725,537 bytes, serve manifest sha256 c7202744...), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file on the host and again in the smoke container; README, engine.py, setup.sh, constraints.txt, serve.patch and the banner image were not used. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry; no outside request for this model). AUTHOR-DEFAULT CONTEXT LIMITS: the engine allows 32,768 tokens for state plus the longest question (strict >) and 65,536 per request and raises an error above that, which the authors' runtime returns through OneJev create_app as HTTP 422, never a truncation; the authors train at 16,384 tokens. The 15 items of 76,374-80,534 tokens were refused exactly as pre-registered and are scored wrong; the largest answered item is 1,854 tokens, inside the trained length. NOT AN AUTHOR-SHIPPED SERVER: the authors ship no server file; our first-party launcher around their DecisionEngine and OneJev's web app is what was timed (see code_revision). SERIAL ONLY: the server handles one request at a time and our driver sent one at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). causal-conv1d is not installed, so transformers' slower torch convolution fallback ran. LICENCE: MIT in the card metadata since 21 Oct 2026 (commit 5494c13e changed 'proprietary' to 'mit' and removed the evaluation-only line); there is no LICENSE file, no copyright holder and no carried Qwen attribution, and the Decision Index row still said 'proprietary' on 22 Oct (stale). Weights are identical to the Decision Index-scored checkpoint (14537f08). Two README drafts briefly named Qwen3.5-27B as base model; the pinned card and the config say Qwen3.5-9B. TRAINING DATA: the card gives a mix of 6.21M decision examples (9.5% public decision data) without naming datasets; the Decision Index states that the training includes train splits of Decision Index benchmarks (author-declared), and that its checks found signs that the model's training data overlapped one private new-domain test, which it does not count for this model; we cannot rule out overlap on other tests (our inference); the authors' technical report separately claims zero contamination with the Jev / Artificial Analysis index benchmarks (self-reported, not verified here); we ran no overlap check and no JevBench use or denial is declared, so whether those Decision Index train splits overlap JevBench sources is unknown to us. SELF-REPORTED, unverified: Decision Index 0.4 score 62.65 ('best model under 15B', tied #8 of 208) and an Artificial Analysis replica score of 65.2 on the card; the DI lab's own Decision Index 0.4.1 full score is 60.26 (public 72.24, equal to the claim by the kit rescore; decision agreement with the submitted rows 99.84% on 819 rows against a 98.5% bar), DI-measured latency median 114 ms. 10 s RULE: max measured decision latency 0.30 s (median 0.08 s, p95 0.10 s); 0 decisions over 10 s; in the smoke a 20k-token request took about 1.9 s warm and an exact 32,768-token request about 9.4 s, but no answered item is that large.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 81.48517777777778,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 61.122200000000014
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 81.48517777777778
      },
      "calibration_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 36.321363985728176,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 19.999999999999996,
        "score": 36.684134375
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 36.321363985728176
      },
      "composite_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 67.3911931820814,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 40.74074074074075,
        "score": 67.93690384615384
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 67.3911931820814
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 84.64020925925927,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 93.92062777777778
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.64020925925927
      },
      "jaggedness": {
       "accuracy": 0.5111111111111111,
       "cc": 25.522811215450304,
       "cc_by_type": {
        "choice": 40.808469682386914,
        "noul": -2.7027027027026973,
        "score": 38.46266666666669
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 25.522811215450304
      },
      "llm_router": {
       "accuracy": 0.805111821086262,
       "cc": 69.18685079356193,
       "cc_by_type": {
        "choice": 71.47133926212908,
        "noul": 71.13402061855669,
        "score": 64.95519250000001
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.18685079356193
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 33.259192963508426,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 25.0,
        "score": 11.890980952380946
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 33.259192963508426
      },
      "multilingual": {
       "accuracy": 0.7127272727272728,
       "cc": 53.590397470200806,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": 36.111111111111114,
        "score": 54.982872300469474
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 53.590397470200806
      },
      "negation_pair_gen": {
       "accuracy": 0.7906976744186046,
       "cc": 50.353867777777765,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 39.99999999999999,
        "score": 17.72826999999999
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 50.353867777777765
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 76.1766397089397,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100.0,
        "score": 55.55694615384616
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.1766397089397
      },
      "usecase": {
       "accuracy": 0.6772727272727272,
       "cc": 48.10202756078433,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 25.0,
        "score": 45.04137680000002
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 48.10202756078433
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 24.111361660561652,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": -39.570676923076945
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.111361660561652
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 1.0,
       "cc": 86.30362000000001,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 58.91086000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 86.30362000000001
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 72.35387820956778,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 75.0,
        "score": 81.88464347826087
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.35387820956778
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.364865642458106,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 99.61415
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.364865642458106
      },
      "en": {
       "accuracy": 0.7141615986099045,
       "cc": 53.212251436303546,
       "cc_by_type": {
        "choice": 69.46583517408865,
        "noul": 40.28776978417265,
        "score": 49.883149350649326
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 53.212251436303546
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.66649451428206,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 30.787411764705887
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.66649451428206
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 50.72039999999999,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 85.49453333333332
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 50.72039999999999
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 76.84436449948402,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100.0,
        "score": 94.06250526315792
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 76.84436449948402
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 70.33820782122905,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 50.0,
        "score": 77.7744
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.33820782122905
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 54.22450476190476,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -37.32648571428572
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 54.22450476190476
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 32.49348811578854,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 39.99999999999999,
        "score": -13.925413333333347
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 32.49348811578854
      },
      "ko": {
       "accuracy": 0.4,
       "cc": 14.908567989417984,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 0.0,
        "score": 33.614592857142846
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 14.908567989417984
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.80816666666665,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 95.09116666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.80816666666665
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 44.05853379509379,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 50.79031999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 44.05853379509379
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 61.86896395463657,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 60.84176279069767
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.86896395463657
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 72.44503948967501,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 59.762932258064524
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.44503948967501
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 37.953801773049655,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 70.45715000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 37.953801773049655
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 54.188839855072466,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": -15.694350000000012
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 54.188839855072466
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.2079,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 61.62370000000001
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.2079
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 10.766112617702452,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -8.94459999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 10.766112617702452
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7653333333333333,
       "cc": 69.68943148548576,
       "cc_by_type": {
        "choice": 69.68943148548576
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.68943148548576
      },
      "noul": {
       "accuracy": 0.7093333333333334,
       "cc": 41.866666666666674,
       "cc_by_type": {
        "noul": 41.866666666666674
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 41.866666666666674
      },
      "score": {
       "accuracy": 0.632,
       "cc": 50.03769070397117,
       "cc_by_type": {
        "score": 50.03769070397117
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.03769070397117
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 18.88307843137255,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -50.0,
        "score": 38.649235294117645
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 18.88307843137255
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 35.477907264957274,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -26.8996115384615
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 35.477907264957274
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 28.424070731707303,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": -15.947300000000041
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 28.424070731707303
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -1.0775926656920127,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": -11.419912500000006
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 26.025824849033814,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 54.88906874999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 26.025824849033814
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": -41.56288291404612,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -241.04085
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 28.68574496157437,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -14.28571428571429,
        "score": 19.95079230769229
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 28.68574496157437
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 53.46004675554209,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 50.0,
        "score": 52.19198695652174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 53.46004675554209
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 35.39760740740741,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 48.69282222222223
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 35.39760740740741
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.24514960850475,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 54.76471363636364
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.24514960850475
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 52.23264363401455,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 34.99250454545454
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 52.23264363401455
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 12.344700044802865,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": 34.8835625
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.344700044802865
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 59.51451296296296,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 58.54353888888888
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.51451296296296
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 75.0689772558171,
       "cc_by_type": {
        "choice": 89.48051948051949,
        "noul": 81.81818181818181,
        "score": 53.90823046875
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.0689772558171
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 10.657687181513685,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 66.52200588235296
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 10.657687181513685
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 62.65915098326311,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 71.35721153846156
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.65915098326311
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 62.77636666666666,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 38.3291
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 62.77636666666666
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 37.59344980676328,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 0.0,
        "score": 41.76585666666666
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 37.59344980676328
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 95.78782850721109,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 98.03964210526316
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 95.78782850721109
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.75459740234666,
      86.4868177301127
     ],
     "I": [
      50.13470277647599,
      60.016870093437255
     ],
     "capability": [
      65.77545984215375,
      72.50826992991473
     ],
     "composite": {
      "A": [
       58.43477071734329,
       62.13098693849953
      ],
      "B": [
       55.76855695551602,
       60.812107579242856
      ],
      "C": [
       40.92376735438138,
       61.95925697779976
      ]
     },
     "seed": 32
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.65640909439074,
     "capability": 70.24682642468954,
     "intelligence": 55.83724375498836
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 78.76261049180326,
      "cc": 69.68943148548576,
      "cc_by_type": {
       "choice": 69.68943148548576
      },
      "cc_tiered": 70.74726414460008,
      "cc_tiered_open": 73.67205398217818,
      "cc_tiered_sealed": 67.82247430702198,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.68943148548576
     },
     "noul": {
      "accuracy": 0.7093333333333334,
      "calibration": 88.76168525469171,
      "cc": 41.866666666666674,
      "cc_by_type": {
       "noul": 41.866666666666674
      },
      "cc_tiered": 44.265706710066624,
      "cc_tiered_open": 51.134575569358184,
      "cc_tiered_sealed": 37.396837850775064,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 41.866666666666674
     },
     "score": {
      "accuracy": 0.632,
      "calibration": 86.44493153667727,
      "cc": 50.03769070397117,
      "cc_by_type": {
       "score": 50.03769070397117
      },
      "cc_tiered": 52.49876041029837,
      "cc_tiered_open": 58.45286032699879,
      "cc_tiered_sealed": 46.544660493597945,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.03769070397117
     }
    },
    "raw_on_A": {
     "calibration": 86.62408266363812,
     "capability": 72.28243588218652,
     "intelligence": 57.94078910073491
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 56.53431921907342,
    "Emphasis on Accuracy 60:20:20": 54.76668226453846,
    "Emphasis on Cost 20:20:60": 51.45393403230081,
    "Emphasis on Speed 20:60:20": 65.05767831289855,
    "Intelligence only": 52.313197148753815
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Fastino (Hugging Face)",
   "axes": {
    "calibration": 93.46414212015411,
    "cost": 30.624280317474202,
    "intelligence": 76.38100890445796,
    "speed": 85.49903308442846
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03759375786907435,
      "ece_score": 92.48124842618512,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 92.48124842618512,
      "score_before_time_limit": 92.48124842618512,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.13746630727762799,
      "brier": 0.05329684013928134,
      "ece": 0.04180203505001547,
      "n": 371,
      "score": 91.63959298999691,
      "score_before_time_limit": 91.63959298999691,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9892183288409704,
      "interval90_width": 0.21406109613656782,
      "n": 371,
      "nrps": 0.03790344886809453,
      "score": 96.2715849442803,
      "score_before_time_limit": 96.2715849442803,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.018332426123149732
     }
    },
    "score": 93.46414212015411
   },
   "capability": 84.92257551230603,
   "composite_ci95": {
    "A": [
     21.69177878208421,
     22.169244285324318
    ],
    "B": [
     22.599928782842603,
     23.371997955589308
    ],
    "C": [
     21.69177878208421,
     22.169244285324318
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string Qwen/Qwen3.8-27B (the card's base_model; listed in the module's base references as qwen/qwen3.8-27b) priced as qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output. Estimated, not charged, not a quote from the author or any host. This row does not use the operator's hosted tariff of the separate hosted rows (see disclosure). Tokens as reported by the usage block of the author's server: input_tokens = the rendered prompt of the one question (chat template and options included); output_tokens = 0 on every row (the server generates nothing: one forward pass and a 255-way linear readout). Of the 1,500 items sent, 1,485 were answered; the server-reported total of 727,293 input tokens over those 1,485 equals our pre-registered offline count (first-party transcription of the author's prompt renderer and the pinned tokenizer, made before the run). The 15 refused items (HTTP 422) carry no usage and count 0 tokens. Those 15 are the 15 standard very long items, which count for no row's cost on the common cost basis (v1.6.30). The 3 items that were sent but are not scored (see disclosure) are outside the scored set and therefore outside this row's cost basis too: the common cost basis of the scored set is 1,482 items with 724,641 input tokens, the same basis the hosted fastino-glide-api row uses.  COST FIGURES (scored n = 1,497): USD 0.205364 per 1,000 decisions on the common cost basis of the scored set (1,482 items: the 1,497 scored items except the 15 very long ones that the Jev reference refused; the same basis the hosted fastino-glide-api row uses); separately, USD 0.203306 per 1,000 decisions over all 1,497 scored rows. The common-basis figure is 3.18x the official cost cap USD 0.064595 (all-rows figure 3.15x). OVER THE OFFICIAL COST CAP, DISCLOSED: the row is ranked (v1.6.16 vllm-sr-d3 precedent) and falls outside the official-caps board views by construction.",
    "common_basis": {
     "n_items": 1482,
     "rule": "common",
     "usd_per_1000_all_items": 0.20330609218436874
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.20536384615384615
   },
   "custody_exclusion": {
    "derived_gold_sha256": "9f1f8200d0a1b4384f43ffc0a91f3f36e4790fc3dbcef7486bf233b0142baa47",
    "exclusion_list_sha256": "31b47af58dce134a70e5cbf60aad40db0000dd398f26649070fb7c9dfd1deeb3",
    "items": 3,
    "rule": "an item a provider has already seen is never scored for that provider; the 3 items were never sent and are not in this row's gold",
    "scored_items": 1497
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1497,
    "treatment": "share"
   },
   "display": "Fastino GLiDE 1.2 (open weights, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.574/h including disk. The authors' server (127.0.0.1:8000) in ONE --network none --read-only container; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before any sealed input (only loopback, no default route, DNS and TCP fail). Measured 23 Oct 2026 (VM reserved 08:58:49Z, scored loop started 09:05:43Z, teardown confirmed 09:10:37Z); this row measurement 277 s, scored loop 227.5 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 6ab4c6deab3cffc6e9fb7f176f52250b20d6ee936dabc9711a9858370447432a. Node-shred receipt verified; exact-id VM teardown confirms the provider instance absent and the guard closed. GPU rental is not the benchmark's estimated model tariff. Earlier attempts of this same row made no sealed contact: attempt 0 failed before any sealed byte on a runner bug (a JSON false in Python source in the import check; fixed and reviewed), and two Verda create calls answered HTTP 503 with no instance; the one exposure use (on file) was consumed by this run.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.71533523123628,
    "I_sealed": 73.04668257767963,
    "base": 76.38100890445796,
    "excess": -3.1708741167990624,
    "gap": 6.668652653556649,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 90.51495356571994,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 86.78829820698331,
       "judge": 96.044825313118,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 75.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 70.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 72.68281196235743,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.58216426950466,
       "hard": 72.46893187870765,
       "judge": 82.88762650645698,
       "standard": 58.35367415993408
      }
     },
     "sealed|choice": {
      "cc": 87.0566039972978,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 82.30600208164681,
       "judge": 86.77766895200784,
       "standard": 91.45749841805527
      }
     },
     "sealed|noul": {
      "cc": 63.1141990537663,
      "n": {
       "easy": 58,
       "hard": 109,
       "judge": 64,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 50.458715596330286,
       "judge": 68.75,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 68.9692446819748,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 66,
       "standard": 75
      },
      "tiers": {
       "easy": 86.472785214709,
       "hard": 64.98331797139856,
       "judge": 69.06085764182093,
       "standard": 68.0519083969909
      }
     }
    }
   },
   "jevbench_score": 22.02118081045893,
   "key": "fastino-glide-1.2-open",
   "last_measured_on": "2026-10-23",
   "licence": "Fastino Community License for Fastino's modifications and weights (LICENSE and NOTICE at the pin; benchmarking is outside 'Production'; attribution line 'Built with Fastino GLiDE' for Commercial Use), Qwen3.8-27B parts Apache-2.0 (licenses/Qwen3.8-27B-LICENSE); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "fastino/GLiDE@a7e45896be7c96ba3354891da790c5ec99c30b6c",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.975,
    "decisive": 320,
    "decisive_rate": 0.862533692722372,
    "n": 373,
    "supported": true,
    "valid": 371
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 132,
   "ranked": true,
   "ranks": {
    "A": 132,
    "B": 125,
    "C": 113,
    "capability": 9
   },
   "repo": "https://huggingface.co/fastino/GLiDE",
   "scores": {
    "A": 22.02118081045893,
    "B": 23.090092184481797,
    "C": 22.02118081045893
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1497,
    "p50_s_adjusted": 0.4293506360000038,
    "p50_s_raw": 0.13967531800000188,
    "p95_s_adjusted": 0.6565753607999796,
    "p95_s_raw": 0.2532876803999898
   },
   "status": {
    "answered_ok": 1482,
    "missing": 0,
    "rows": 1497,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run as shipped: fastino/GLiDE at a7e45896be7c96ba3354891da790c5ec99c30b6c (Hub main at the pin; the five glide/*.py files are served unmodified from the hash-verified snapshot folder, sha256 __init__.py 7e807d7d94895d436c43c2b2add0521608e208c5dde7e0016549c5b8ee5d7192, images.py 4858f30b92819abe463d845886f3240145c532fd0dfc725f5a08586e7591c616, model.py dd771ffe6c354e4a46dd1e4d8573be4faf0e8ca416010757dd2de4e5724172b2, prompt.py e47149ac980fe12c3cb24ea0ee63822d5bcde71f32d4a5cc9f37499853948338, server.py baa81296446828455b0ca1b43e58f3662b7a06ab684e5ef02d0e4fa77574030a; re-hashed inside the sealed container). Command: env PYTHONPATH=<snapshot> PYTHONDONTWRITEBYTECODE=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 USE_HUB_KERNELS=NO GLIDE_MODEL_NAME=glide-v1.2 GLIDE_MAX_BATCH=1 GLIDE_MAX_LEN=40000 GLIDE_API_KEY= python3 -m glide.server --model <snapshot> --host 127.0.0.1 --port 8000. bf16, PyTorch SDPA attention, one request per forward pass, temperature 1.0 from decision_config.json (sha256 df40c569d6248a8562177ea8377d47c6fbe558521f60dd14e4e8a78d04e7ce10), no kernel packages (flash-linear-attention, causal-conv1d and Hub kernels absent; checked in the container). Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 used only as a CUDA 13.0 / PyTorch base (vLLM is not used); derived image sha256:f8d7c68d21d235430e49996d047c1175da52a16841fddca82fbe21fbd662f9ce, built offline from four hash-pinned wheels (transformers 5.17.0, huggingface-hub 1.33.0, fastapi 0.136.3, uvicorn 0.54.0). Python 3.12.3, torch 2.13.0+cu130, tokenizers 0.23.2, numpy 2.2.6. DEVIATIONS from the author's requirements.txt, asserted before the run: torch 2.13.0 (author 2.14.0), numpy 2.2.6 (2.5.3), fastapi 0.136.3 (0.142.2). Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: one dormant Hub download line, snapshot_download at glide/model.py:216-218, reached only if --model is not a directory (we pass the local snapshot folder); dormant Hub-kernel loaders in the transformers Qwen3.5 code, inert with USE_HUB_KERNELS=NO and no kernels package. No trust_remote_code, subprocess, eval/exec, pickle, telemetry or request-body logging in the author code. The source and runner were reviewed before the run, findings were fixed and the final changes rechecked. Weights: Fastino Community License, public, ungated, 27 files, 52,193,616,460 bytes, every file hash-checked on the node (VERIFY OK 27 files); manifest sha256 5e9fdbcf53ffd2ae843d025e62745a145672283a0bada42bc3e90779d277d0de. The Hub main moved to e827fc56 on 23 Oct 2026 07:12Z ('Update README.md'; only README.md changed, one line making a LICENSE link absolute); the measurement stays at the pinned commit a7e45896. FIRST MEASUREMENT OF THE OPEN WEIGHTS, A NEW ROW (key fastino-glide-1.2-open; evaluator exposure and device ruling). NOT TO BE CONFUSED WITH THE TWO HOSTED ROWS: fastino-glide (hosted API, first measured 6 Oct 2026 in v1.6.1 on 600 items) and fastino-glide-api (hosted API, 22 Oct 2026 on 1,497 items; Fastino's API reports its model as glide_v2). This row is the open-weights checkpoint fastino/GLiDE at a7e45896 (version 1.2.0, engine name glide-v1.2, first uploaded 19 Oct 2026), served by us on our own rented GPU through the authors' own server; as of 22 Oct 2026 nothing published says that either hosted row served this checkpoint, and the hosted rows differ in prompt limit (65,536 vs 40,000 tokens), tariff and engine name. SCORED ON 1,497 OF THE DRAW'S 1,500 ITEMS: all 1,500 items were SENT unchanged (input hash unchanged), but 3 items Fastino had already seen through its hosted GLiNER-2.5-Decide on 8 Oct 2026 are never scored for this row (exposure rule: an item a provider has seen is never scored for that provider). The row is scored with a second invocation of the unmodified frozen scorer on a derived gold that lacks exactly those 3 ids (exclusion file sha256 31b47af58dce134a70e5cbf60aad40db0000dd398f26649070fb7c9dfd1deeb3; the derived gold's sha256 is recorded in the release proof); the other rows of this board are scored on all 1,500 items. NOT ANSWERED, counted wrong: 15 items exceed the author's 40,000-token context limit (refused above 39,992 prompt tokens; HTTP 422 'prompt exceeds the maximum context length'), exactly the 15 standard very long items (76,339-80,497 tokens) at positions 65, 169, 474, 482, 515, 573, 580, 729, 833, 899, 951, 1057, 1116, 1198, 1418, as pre-registered by id and position; never truncated. Answered: 1,485 of the 1,500 items sent with HTTP 200 (status counts 200: 1,485, 422: 15); of the 1,497 scored items, 1,482 were answered and the same 15 refused. AUTHOR'S OWN SERVER on our rented hardware, bf16, ONE REQUEST PER FORWARD PASS (GLIDE_MAX_BATCH=1). NATIVE OUTPUTS: choice argmax and probabilities, Noul P(true), score level and expectation, all from the 255-way readout softmax at temperature 1.0; no rotation or calibration file, nothing generated. OPTIONAL KERNELS OFF, FIDELITY RISK: the author's validation and all numbers in its model card used flash-linear-attention; this run used the PyTorch fallback, which the author says 'was not part of the validation', so small bf16 differences in probabilities against the author's own runs are possible and were not measured. The reason was the 10-second rule: a cold Triton cache adds 8-35 s per new shape according to the model card and the sealed container starts empty. Three package versions differ from the author's pins (see code_revision). AUTHOR EXPOSURE: the model card at the pinned revision makes no statement about training data, JevBench or decontamination; no overlap or contamination audit was run by us, and no author statement was requested or received. LICENCE NOTES: the Fastino Community License excludes evaluation, testing, benchmarking and research from 'Production' (section 6(b)); its section 4 asks for the line 'Built with Fastino GLiDE' on a related website for any Commercial Use, and the model card's introduction says 'Internal use only.' Built with Fastino GLiDE. This result does not establish that the hosted fastino-glide-api row uses the same weights. The model was under Apache-2.0 for about 50 minutes on 22 Oct 2026 before the licence changed; weights, code and configuration are byte-identical in all five revisions up to the pin (only README.md and licence files changed). 10 s RULE: 0 decisions over 10 s, maximum measured decision latency 0.485 s over the 1,500 requests, so the 10-second limit of the v1.7 share view costs this row nothing (the frozen v1.6 and v1.7-off views apply none). The longest answered prompt was 1,839 tokens (pre-registered offline count). For context, the pre-run synthetic probes (not benchmark items) took 5.3 s at 16,000 tokens and 17.06 s at the 39,992-token limit. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. CONTEXT: author-imposed limit 40,000 tokens (GLIDE_MAX_LEN, default); no truncation. The vendor's reported figures and the Decision Index figure for this checkpoint are not independently verified by us.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 93.28683402489601,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 88.19383540802139
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 93.28683402489601
      },
      "calibration_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 61.10934993664238,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 43.99999999999999,
        "score": 64.77873909942872
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.10934993664238
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 88.20348809055965,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 75.26990419941157
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 88.20348809055965
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.29867459427453,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 98.89602378282356
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.29867459427453
      },
      "jaggedness": {
       "accuracy": 0.7944444444444444,
       "cc": 58.71369167107085,
       "cc_by_type": {
        "choice": 79.78825794032724,
        "noul": 18.918918918918926,
        "score": 77.43389815396637
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.71369167107085
      },
      "llm_router": {
       "accuracy": 0.9552715654952076,
       "cc": 88.30660149539369,
       "cc_by_type": {
        "choice": 97.62261160517741,
        "noul": 100.0,
        "score": 67.29719288100368
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 88.30660149539369
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.75,
       "cc": 52.56750812051602,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 33.33333333333333,
        "score": 49.11145906945185
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 6,
        "score": 7
       },
       "score": 52.56750812051602
      },
      "multilingual": {
       "accuracy": 0.8181818181818182,
       "cc": 69.36083093459378,
       "cc_by_type": {
        "choice": 84.3495272253016,
        "noul": 55.55555555555556,
        "score": 68.17741002292419
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.36083093459378
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 74.46799128621208,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 73.33333333333334,
        "score": 50.07064052530291
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.46799128621208
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 82.54058281009027,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 80.0,
        "score": 79.63376044228282
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.54058281009027
      },
      "usecase": {
       "accuracy": 0.8545454545454545,
       "cc": 73.9863191228161,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 58.33333333333333,
        "score": 73.42954560374241
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 73.9863191228161
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 49.037941029788264,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50.0,
        "score": 16.16144213698383
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 49.037941029788264
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 55.383081739866704,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 32.81591188626678
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 55.383081739866704
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 90.75212945562821,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 88.18559190670764
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.75212945562821
      },
      "el": {
       "accuracy": 0.9333333333333333,
       "cc": 94.3823560381911,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100.0,
        "score": 99.90684465088616
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 94.3823560381911
      },
      "en": {
       "accuracy": 0.8748913987836664,
       "cc": 76.71598640342684,
       "cc_by_type": {
        "choice": 88.79681722214764,
        "noul": 69.06474820143885,
        "score": 72.28639378669403
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 76.71598640342684
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 73.96984447427957,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 31.69746164469842
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.96984447427957
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.72662140938952,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.84653089483527
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.72662140938952
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 92.11359084699383,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 97.51724312921681
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 92.11359084699383
      },
      "hi": {
       "accuracy": 0.8888888888888888,
       "cc": 70.43840994806452,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 66.66666666666667,
        "score": 44.64856317752691
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 4
       },
       "score": 70.43840994806452
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 66.79664710844526,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 14.41955940795463
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 66.79664710844526
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 52.487729250622465,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 37.4631877518674
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 52.487729250622465
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 18.81075830924279,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 56.43227492772836
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.81075830924279
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.9712888346815,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.91386650404452
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.9712888346815
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 1.9262230233236461,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -34.91397162266976
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.9262230233236461
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 78.59736445947553,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 71.02696430521456
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 78.59736445947553
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 78.12012789523055,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 66.18124402747125
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 78.12012789523055
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 60.220633886041526,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 46.19381655174158
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 60.220633886041526
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 86.05162469672574,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 79.8940045249598
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 86.05162469672574
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.92582991138106,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 84.03555425027218
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.92582991138106
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 65.47290998186114,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 63.08539661225008
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 65.47290998186114
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.908,
       "cc": 88.11687938919613,
       "cc_by_type": {
        "choice": 88.11687938919613
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 88.11687938919613
      },
      "noul": {
       "accuracy": 0.8364611260053619,
       "cc": 67.29222520107238,
       "cc_by_type": {
        "noul": 67.29222520107238
       },
       "low_n": false,
       "n": 373,
       "n_by_type": {
        "noul": 373
       },
       "score": 67.29222520107238
      },
      "score": {
       "accuracy": 0.8021390374331551,
       "cc": 70.23031955117594,
       "cc_by_type": {
        "score": 70.23031955117594
       },
       "low_n": false,
       "n": 374,
       "n_by_type": {
        "score": 374
       },
       "score": 70.23031955117594
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 28.69392249485216,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -25.0,
        "score": 43.08176748455649
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 28.69392249485216
      },
      "code_linting": {
       "accuracy": 1.0,
       "cc": 93.48129249546712,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 80.44387748640132
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 93.48129249546712
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 63.834484500614856,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 24.0237787050966
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 63.834484500614856
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": -0.8645333531519933,
       "cc_by_type": {
        "choice": 53.21637426900585,
        "noul": 33.33333333333333,
        "score": -89.14330766179515
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.875,
       "cc": 68.38791810434252,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 86.32317460288262
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 68.38791810434252
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 56.21750624743703,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 47.89780176117902
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 56.21750624743703
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 51.137965239512276,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -14.28571428571429,
        "score": 67.69961000425111
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.137965239512276
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 52.67279304179414,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 58.01837912538241
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 52.67279304179414
      },
      "insurance_claims": {
       "accuracy": 0.6428571428571429,
       "cc": 62.805898885724105,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 50.0,
        "score": 59.25102999050564
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 4
       },
       "score": 62.805898885724105
      },
      "knowledge_graphs": {
       "accuracy": 0.9545454545454546,
       "cc": 90.58078803635563,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 71.74236410906687
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 90.58078803635563
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.07503322126443,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 57.297451084981944
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.07503322126443
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 34.89585122275884,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 68.12841388333031
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 34.89585122275884
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 80.84937867696657,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 82.54813603089968
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 80.84937867696657
      },
      "model_routing": {
       "accuracy": 0.9351851851851852,
       "cc": 84.84238463865871,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 100.0,
        "score": 58.033647422469684
       },
       "low_n": false,
       "n": 216,
       "n_by_type": {
        "choice": 101,
        "noul": 65,
        "score": 50
       },
       "score": 84.84238463865871
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 54.98952694592623,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0.0,
        "score": 88.00121039631993
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 54.98952694592623
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 67.86249205295886,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 70.25414282554324
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 67.86249205295886
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 62.175293189844275,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 36.52587956953282
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 62.175293189844275
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 55.986205509765625,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 63.610790442340345
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 55.986205509765625
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.92032149163197,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.7609644748959
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.92032149163197
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      89.20818030834448,
      93.61993386336131
     ],
     "I": [
      72.30102290134639,
      79.94449290262376
     ],
     "capability": [
      81.44074129934772,
      86.23327088510355
     ],
     "composite": {
      "A": [
       21.69177878208421,
       22.169244285324318
      ],
      "B": [
       22.599928782842603,
       23.371997955589308
      ],
      "C": [
       21.69177878208421,
       22.169244285324318
      ]
     },
     "seed": 16
    },
    "ci95_n": 1197,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1497,
    "n_public": 300,
    "n_sealed_side": 1197,
    "on_S": {
     "calibration": 93.46414212015411,
     "capability": 84.92257551230603,
     "intelligence": 76.38100890445796
    },
    "per_type": {
     "choice": {
      "accuracy": 0.908,
      "calibration": 92.48124842618512,
      "cc": 88.11687938919613,
      "cc_by_type": {
       "choice": 88.11687938919613
      },
      "cc_tiered": 88.78577878150887,
      "cc_tiered_open": 90.51495356571994,
      "cc_tiered_sealed": 87.0566039972978,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 88.11687938919613
     },
     "noul": {
      "accuracy": 0.8364611260053619,
      "calibration": 91.63959298999691,
      "cc": 67.29222520107238,
      "cc_by_type": {
       "noul": 67.29222520107238
      },
      "cc_tiered": 69.5312196096989,
      "cc_tiered_open": 75.94824016563147,
      "cc_tiered_sealed": 63.1141990537663,
      "low_n": false,
      "n": 373,
      "n_by_type": {
       "noul": 373
      },
      "score": 67.29222520107238
     },
     "score": {
      "accuracy": 0.8021390374331551,
      "calibration": 96.2715849442803,
      "cc": 70.23031955117594,
      "cc_by_type": {
       "score": 70.23031955117594
      },
      "cc_tiered": 70.82602832216611,
      "cc_tiered_open": 72.68281196235743,
      "cc_tiered_sealed": 68.9692446819748,
      "low_n": false,
      "n": 374,
      "n_by_type": {
       "score": 374
      },
      "score": 70.23031955117594
     }
    },
    "raw_on_A": {
     "calibration": 89.83389322509132,
     "capability": 82.01303424195552,
     "intelligence": 74.19217525881973
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.00536193029490617,
     "n": 373
    },
    "score": {
     "invalid_rate": 0.008021390374331583,
     "n": 374
    }
   },
   "views": {
    "Balanced 33:33:33": 19.592168922037885,
    "Emphasis on Accuracy 60:20:20": 22.42937120079969,
    "Emphasis on Cost 20:20:60": 15.280622314581365,
    "Emphasis on Speed 20:60:20": 23.204200566619555,
    "Intelligence only": 28.653466120740877
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tattrongvu (Hugging Face)",
   "axes": {
    "calibration": 91.73503889029561,
    "cost": 42.76241306716163,
    "intelligence": 71.63230144667592,
    "speed": 84.65261351124272
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.028876069431341744,
      "ece_score": 94.22478611373165,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.96845563221524,
      "score_before_time_limit": 94.22478611373165,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.1253333333333333,
      "brier": 0.07307799064054041,
      "ece": 0.04494305379107265,
      "n": 375,
      "score": 90.52599516582929,
      "score_before_time_limit": 91.01138924178547,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9386666666666666,
      "interval90_width": 0.17200000000000007,
      "n": 375,
      "nrps": 0.04654031677107972,
      "score": 91.71066587284233,
      "score_before_time_limit": 92.45026801697816,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.05222716144467865
     }
    },
    "score": 91.73503889029561
   },
   "capability": 81.68367016848578,
   "composite_ci95": {
    "A": [
     47.618529604462914,
     49.21056610344301
    ],
    "B": [
     47.87876685138602,
     50.285269904771376
    ],
    "C": [
     47.618529604462914,
     49.21056610344301
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string google/gemma-4-31B-it priced as google/gemma-4-31b-it USD 0.09/M input, USD 0.34/M output. The exact case of 31B is required by the frozen module; it resolves to the lowercase google/gemma-4-31b-it listing at USD 0.09/M input and USD 0.34/M output, as required by the frozen pricing policy. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author engine through the shim: input_tokens includes the rendered prompt, chat template and options; output_tokens = 0 is our shim's labelled convention because the engine reports only input usage. The 31b max_tokens=1 readout is not counted as output. Reported input usage is 2,525,873 tokens: 1,969,524 first-read tokens plus 556,349 tokens from 581 conditional rereads. The offline check has offset 0 and 0 item-level mismatches, within the preregistered band 1,969,524-3,459,523. The 15 standard long items contribute 1,191,112 first-read tokens; no reread of those items fits. Excluding them as for every row on the common 1,485-item cost basis (v1.6.30) leaves 1,334,761 reported input tokens, INCLUDING the 556,349 reread tokens, and 0 output tokens. The preregistered all-rereads common-basis bound (2,268,411 tokens, USD 0.137479/1,000) is not the measured usage. Measured arithmetic: USD 0.080894606 per 1,000 decisions on the common basis = 1.252342099x the unchanged USD 0.06459465517241379 cap; separately USD 0.151552380/1,000 over all 1,500 items = 2.346206193x. OVER COST CAP: ranked_over_cap, shown only in uncapped views and excluded from official cost-capped views; no cap exception or top-up (ruling on file).  COST ON THE COMMON BASIS: USD 0.080895 per 1,000 decisions on 1,485 items, 1.252 times the official cost cap USD 0.064595; estimated, not charged. OVER THE OFFICIAL COST CAP, DISCLOSED: ranked_over_cap; outside the official-caps views.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.15155237999999996
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.08089460606060606
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jeffan-31b-1710",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5741917808219177/h recorded by the runner. One VM for both rows back to back, 31b then 4b. Engine and loopback shim in a --network none --read-only container per row; serial requests, read-only code/model/input, blank credential variables, HF offline flags. In-container network cut proven before sealed input: only loopback, no default route, DNS and TCP fail; code inventory/hashes and model files checked. Attempt 2 measured 24 Oct 2026 07:59-08:36Z; this row's measurement container 1035 s, run cap 1945 s, complete=true, deadline_hit=false, rc 0; scored loop 862.812179 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 5f578c036684557641cbfa55bfa043ed209775cbf793d05c4f78c1f4ce7c4fe0. Row cleanup verified; final exact-id VM teardown confirms provider absent and guard closed at 08:36:06Z. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.99205442745344,
    "I_sealed": 70.27254846589841,
    "base": 71.63230144667592,
    "excess": -7.120020808800682,
    "gap": 2.719505961555029,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 78.4827278527896,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 83.01352626612143,
       "judge": 72.31377719182598,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 69.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 70.54519526393928,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.47787147888829,
       "hard": 77.29517951037028,
       "judge": 75.12442806811568,
       "standard": 52.21003945733818
      }
     },
     "sealed|choice": {
      "cc": 84.85434539262528,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 83.34682548860874,
       "judge": 85.01469147894223,
       "standard": 82.91499683611053
      }
     },
     "sealed|noul": {
      "cc": 60.194263790764055,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 49.09090909090909,
       "judge": 60.00000000000001,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 65.76903621430593,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.39945205262151,
       "hard": 64.68166177839782,
       "judge": 62.22278902181544,
       "standard": 63.94794795570004
      }
     }
    }
   },
   "jevbench_score": 48.71528103120407,
   "key": "jeffan-31b-1710",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card and LICENSE); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tattrongvu/Jeffan-31b-1710@cebfa86c2bed2b5c83b12e44de4374b20c87fd79",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9447852760736196,
    "decisive": 326,
    "decisive_rate": 0.8739946380697051,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 75,
   "ranked": true,
   "ranks": {
    "A": 75,
    "B": 71,
    "C": 60,
    "capability": 22
   },
   "repo": "https://huggingface.co/tattrongvu/Jeffan-31b-1710",
   "scores": {
    "A": 48.71528103120407,
    "B": 49.40936153760858,
    "C": 48.71528103120407
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.39501961600026336,
    "p50_s_raw": 0.12250980800013167,
    "p95_s_adjusted": 0.8672014321001599,
    "p95_s_raw": 0.3586007160500799
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN DECISION INDEX ENGINE, run unmodified: tattrongvu/Jeffan-31b-1710 at cebfa86c2bed2b5c83b12e44de4374b20c87fd79, three author package files (__init__.py, engine.py, render.py), the three DI-kit base files at f8552a13b643d39f61185f5951bb5fed16bedf51 and our transport-only jeffan_shim.py. Exact ten-file code manifest sha256 f38f38d69571180283539bb9d8fad4412a6376c5d010bbe53919f8a81a5b8004, hash-checked in the container. dm_decision_engine.DecisionEngine(max_len=98304, max_batched_tokens=4096, reread_below=0.9, gpu_mem=0.85), vLLM 0.29.0 bf16, prefix caching, causal attention; conditional read-twice when first-read top-1 probability is below 0.9 and the doubled prompt fits. The predeclared memory ladder was not used. Python 3.12.3, torch 2.13.0+cu130, transformers 5.20.0, tokenizers 0.23.3, huggingface-hub 1.33.0. The author verified this stack on B200 only; this measures RTX PRO 6000. Base image vllm/vllm-openai:v0.29.0@sha256:c2914767605584b6d8f45686b82de173ecc99e781897aa3d0a66dacd72c51ae1; derived image sha256:b911dd6884706894279758c428e1a8884d4b995371fc4cf09c89136ee825fffd, hash-pinned wheel overlay. The submitted code was reviewed and cleared at the exact pins with the reviewed safeguards; evaluator exposure and the device were cleared for both attempts. Source review records dormant Hub-capable hf_hub_download/from_pretrained paths; the model argument is a hash-checked local directory, revision=None, offline flags set and network disabled. No author shell/subprocess, eval/exec, pickle or trust_remote_code path on the reviewed serving path. The 4b vLLM plugin was excluded and never executed. The runner and device selection change were reviewed; the amended runner was rechecked with the suite rerun and amendment recorded before retry. Weights: Apache-2.0 at this pin (earlier CC BY-NC 4.0 pins are not measured), public, ungated; bf16, 31,273,086,512 parameters, 62,546,338,264 weight bytes.  FIRST MEASUREMENT of Jeffan-31b-1710 (our own intake). AUTHOR'S OWN ENGINE on our rented hardware, one request at a time; our harness replaces the author's Decision Index pipeline. NATIVE OUTPUTS: next-token probabilities over the option codes, choice argmax and probabilities, noul P(true), score expectation and probabilities; the first-party transport shim adds no prompt text or probability arithmetic. AUTHOR EXPOSURE: the preregistration records training data as undisclosed (Decision Index note) and author-reported Decision Index suite figures; no overlap or contamination audit was run by us. The author's reported figures are not independently verified by us. CONTEXT: engine max_len 98,304 prompt tokens; no lower evaluator-imposed cap and no truncation. All 1,500 items answered HTTP 200, 0 refusals; longest first read 81,715 tokens. The card states inputs up to 8,192 tokens and the author validated at most 32,362 first-read tokens; these long items exceed that validation. 10 s RULE: exactly 15 decisions over 10 s, the standard long items at positions in the pre-registered long-item set; these count wrong in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 44.157234 s, median 0.122518 s, p95 0.358708 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. ATTEMPT HISTORY: attempt 1, pod window 24 Oct 2026 05:52-06:21Z (about USD 1.24) on a separate disposable GPU instance, stopped the 31b row after 1,057 of 1,500 items at the runner's own time cap. The smoke under-predicted long-item latency because repeated probe prompts hit the prefix cache. The partial is retained as a receipt only, never scored or merged; the 4b row had not started. Amendment 3 corrected the first-party runner to use cold nonce-bearing probes, maximum anchor latency and a long-item budget floor; a further review preceded the retry. evaluator exposure and device ruling admitted the fresh full 31b run as the final same-use retry; 4b remained a first use. Attempt 2 is this measurement, both rows back to back on new a separate disposable GPU instance, 24 Oct 07:59-08:36Z, about USD 1.57 rental. Between attempts only our first-party runner (smoke probes and fit gate, pre-registration Amendment 3) changed; the scoring harness, model, author code, input and engine settings are unchanged, and the pinned image stack was rebuilt on the new VM. OVER COST CAP: measured USD 0.080894606 per 1,000 decisions on the common basis = 1.252342099x; ranked_over_cap, uncapped views only (ruling on file). ",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1.0,
       "cc": 90.78364769739426,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 72.3509430921828
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 90.78364769739426
      },
      "calibration_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 70.4990841343402,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 52.0,
        "score": 75.40393320895905
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 70.4990841343402
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 85.83224203573293,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 77.77777777777777,
        "score": 82.97098084974627
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.83224203573293
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.78298898647171,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 97.34896695941512
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.78298898647171
      },
      "jaggedness": {
       "accuracy": 0.8,
       "cc": 65.21005041341623,
       "cc_by_type": {
        "choice": 78.34456207892205,
        "noul": 40.54054054054055,
        "score": 76.7450486207861
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 65.21005041341623
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 80.6688597599484,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 93.81443298969072,
        "score": 57.70169986944481
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.6688597599484
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 49.27195296946737,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 25.0,
        "score": 47.5581269496392
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 49.27195296946737
      },
      "multilingual": {
       "accuracy": 0.7709090909090909,
       "cc": 59.75363667787247,
       "cc_by_type": {
        "choice": 79.45875448320835,
        "noul": 36.111111111111114,
        "score": 63.69104443929796
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 59.75363667787247
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 76.39048614552513,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 73.33333333333334,
        "score": 62.50479176990871
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.39048614552513
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 86.47973622784764,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 71.4512206955549
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.47973622784764
      },
      "usecase": {
       "accuracy": 0.7909090909090909,
       "cc": 68.50358527761284,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 62.5,
        "score": 66.2950695583287
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 68.50358527761284
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 57.917196734831215,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": -7.200790747887287
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 57.917196734831215
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 77.78630484073538,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 33.35891452220612
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 77.78630484073538
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 76.66453093859816,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 78.88739812552899
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 76.66453093859816
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.58604847392802,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.94436516107639
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.58604847392802
      },
      "en": {
       "accuracy": 0.8549087749782798,
       "cc": 75.53842125405133,
       "cc_by_type": {
        "choice": 85.50176346395575,
        "noul": 70.50359712230217,
        "score": 70.60990317589604
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.53842125405133
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 71.24640609810298,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 23.52714651616864
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.24640609810298
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.5130044521433,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 98.99256526899825
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.5130044521433
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 88.65781563933801,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 87.14991750624928
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 88.65781563933801
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 76.43656310769823,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 50.0,
        "score": 79.30968932309472
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 76.43656310769823
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 68.9658307236104,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 20.927110253450067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.9658307236104
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 44.59217191994967,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 13.776515759849017
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 44.59217191994967
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 12.72898872647921,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 82.63141062388209
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 12.72898872647921
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.96095473924714,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 94.54953088440809
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.96095473924714
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -14.965639046318811,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 7.051134809095516
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 67.40120233664327,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 69.82104247332381
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.40120233664327
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 58.32982292531735,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 57.417282564991545
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 58.32982292531735
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 48.6264307338863,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 51.41120709527591
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 48.6264307338863
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 70.95545468890508,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 12.866364066715242
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 70.95545468890508
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.94549529754356,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 75.09455040875973
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.94549529754356
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 38.564498140494244,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 49.02682775481608
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 38.564498140494244
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8786666666666667,
       "cc": 84.32805832488184,
       "cc_by_type": {
        "choice": 84.32805832488184
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.32805832488184
      },
      "noul": {
       "accuracy": 0.8213333333333334,
       "cc": 64.26666666666667,
       "cc_by_type": {
        "noul": 64.26666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 64.26666666666667
      },
      "score": {
       "accuracy": 0.7733333333333333,
       "cc": 67.67374817243945,
       "cc_by_type": {
        "score": 67.67374817243945
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.67374817243945
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 33.21690881679642,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 0.0,
        "score": 31.650726450389254
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 33.21690881679642
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 84.32387200910605,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 71.60515639998897
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 84.32387200910605
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 29.066115138365063,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50.0,
        "score": 2.2389958215992434
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 29.066115138365063
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -3.1361152440208784,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": -17.595480234986603
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 35.210332375925105,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 67.94983770748546
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 35.210332375925105
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 33.63873931643062,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 33.33333333333333,
        "score": 5.318733672562326
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.63873931643062
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 53.405528031692114,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 55.7347913779895
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 53.405528031692114
      },
      "gaming": {
       "accuracy": 0.8181818181818182,
       "cc": 66.48399365709115,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50.0,
        "score": 70.35790431622114
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 66.48399365709115
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 38.547249746550726,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 58.141749239652164
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 38.547249746550726
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 81.83766752199955,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100.0,
        "score": 58.360968304756675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 81.83766752199955
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 64.59063160789553,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 27.62202402265299
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 64.59063160789553
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 12.068318831643523,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -0.35418329001566384
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.068318831643523
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 64.85792290166587,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 74.57376870499763
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 64.85792290166587
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 76.00451100954336,
       "cc_by_type": {
        "choice": 88.31168831168831,
        "noul": 93.93939393939394,
        "score": 45.762450777547855
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 76.00451100954336
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 24.165403800315858,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 84.0125261802182
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 24.165403800315858
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 62.22189994439916,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 86.75855034387531
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.22189994439916
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 42.19246157222563,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50.0,
        "score": 20.912360086135028
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 42.19246157222563
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 44.21569095834614,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 0.0,
        "score": 61.63258012141524
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 44.21569095834614
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.90957867741692,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.72873603225078
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.90957867741692
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.62696026019485,
      92.3898122960135
     ],
     "I": [
      66.74809318659113,
      75.34912722976662
     ],
     "capability": [
      77.77844533915831,
      83.22563961157051
     ],
     "composite": {
      "A": [
       47.618529604462914,
       49.21056610344301
      ],
      "B": [
       47.87876685138602,
       50.285269904771376
      ],
      "C": [
       47.618529604462914,
       49.21056610344301
      ]
     },
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.73503889029561,
     "capability": 81.68367016848578,
     "intelligence": 71.63230144667592
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8786666666666667,
      "calibration": 92.96845563221524,
      "cc": 84.32805832488184,
      "cc_by_type": {
       "choice": 84.32805832488184
      },
      "cc_tiered": 81.66853662270744,
      "cc_tiered_open": 78.4827278527896,
      "cc_tiered_sealed": 84.85434539262528,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.32805832488184
     },
     "noul": {
      "accuracy": 0.8213333333333334,
      "calibration": 90.52599516582929,
      "cc": 64.26666666666667,
      "cc_by_type": {
       "noul": 64.26666666666667
      },
      "cc_tiered": 65.07125197819776,
      "cc_tiered_open": 69.94824016563147,
      "cc_tiered_sealed": 60.194263790764055,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.26666666666667
     },
     "score": {
      "accuracy": 0.7733333333333333,
      "calibration": 91.71066587284233,
      "cc": 67.67374817243945,
      "cc_by_type": {
       "score": 67.67374817243945
      },
      "cc_tiered": 68.1571157391226,
      "cc_tiered_open": 70.54519526393928,
      "cc_tiered_sealed": 65.76903621430593,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.67374817243945
     }
    },
    "raw_on_A": {
     "calibration": 88.93954523997444,
     "capability": 80.14320664914894,
     "intelligence": 71.34686805832344
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 44.63851874759102,
    "Emphasis on Accuracy 60:20:20": 47.44832029736161,
    "Emphasis on Cost 20:20:60": 38.12483306152971,
    "Emphasis on Speed 20:60:20": 50.247856527870574,
    "Intelligence only": 52.39541741867261
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tattrongvu (Hugging Face)",
   "axes": {
    "calibration": 87.9034951152282,
    "cost": 64.1356069476132,
    "intelligence": 40.09995458474205,
    "speed": 93.04878911983016
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04890108419083565,
      "ece_score": 90.21978316183286,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.21978316183286,
      "score_before_time_limit": 90.21978316183286,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3699731903485255,
      "brier": 0.14739969780221118,
      "ece": 0.08497512533164059,
      "n": 373,
      "score": 83.00497493367189,
      "score_before_time_limit": 83.00497493367189,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9543010752688172,
      "interval90_width": 0.4735663082437274,
      "n": 372,
      "nrps": 0.09846026008120294,
      "score": 90.48572725017985,
      "score_before_time_limit": 90.48572725017985,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04591259745760014
     }
    },
    "score": 87.9034951152282
   },
   "capability": 64.00172484998512,
   "composite_ci95": {
    "A": [
     28.515963494686925,
     51.612621167968996
    ],
    "B": [
     24.89561268129345,
     46.92006592201432
    ],
    "C": [
     19.802752426865915,
     35.842098033311814
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string Qwen/Qwen3.5-4B priced as Qwen/Qwen3.5-4B USD 0.03/M input, USD 0.15/M output. The module selects the deprecated-after-cutoff DeepInfra reference retained under the frozen policy (price snapshots of 25 Sep 2026), USD 0.03/M input and USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author engine through the shim: input_tokens includes the rendered prompt, chat template and options; output_tokens = 0 is our shim's labelled convention because the engine reports only input usage. Reported usage is 776,399 input tokens and 0 output tokens, equal to the preregistered offline single-read count, offset 0 and 0 item-level mismatches. The 15 refused items carry no usage and count 0 tokens. Those are the standard long items excluded from every row's common 1,485-item cost basis (v1.6.30), so common-basis input usage is also 776,399, with 0 output tokens. Measured arithmetic: USD 0.015684828 per 1,000 decisions on the common basis = 0.242819290x the unchanged USD 0.06459465517241379 cap; separately USD 0.015527980/1,000 over all 1,500 items = 0.240391097x.  COST ON THE COMMON BASIS: USD 0.015685 per 1,000 decisions on 1,485 items, 0.243 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01552798
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015684828282828283
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jeffan-4B-2010",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5741917808219177/h recorded by the runner. One VM for both rows back to back, 31b then 4b. Engine and loopback shim in a --network none --read-only container per row; serial requests, read-only code/model/input, blank credential variables, HF offline flags. In-container network cut proven before sealed input: only loopback, no default route, DNS and TCP fail; code inventory/hashes and model files checked. Attempt 2 measured 24 Oct 2026 07:59-08:36Z; this row's measurement container 94 s, run cap 183 s, complete=true, deadline_hit=false, rc 0; scored loop 50.604333 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 e32cbcf090bccf4750a0225b30d9cd73aa0636be787f9f517d3e97946691121a. Row cleanup verified; final exact-id VM teardown confirms provider absent and guard closed at 08:36:06Z. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 47.662298511076926,
    "I_sealed": 32.537610658407175,
    "base": 40.09995458474205,
    "excess": 5.2851610823140405,
    "gap": 15.124687852669751,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 72.15914308499305,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 58.47750865051904,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 31.68391994478951,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 0.0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 39.14383250344822,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.06727265030784,
       "hard": 41.445405969054626,
       "judge": 42.37827155697271,
       "standard": 19.22730691851886
      }
     },
     "sealed|choice": {
      "cc": 62.82197435013218,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 58.36706372152191,
       "judge": 59.4515181194907,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 1.5438817487196292,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 9.090909090909083,
       "judge": -35.38461538461538,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 33.24697587636973,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 56.57048051927891,
       "hard": 29.398575584954646,
       "judge": 30.460231180970943,
       "standard": 33.462141180843474
      }
     }
    }
   },
   "jevbench_score": 41.06456220687438,
   "key": "jeffan-4b-2010",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card and LICENSE); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tattrongvu/Jeffan-4B-2010@937a008f86f4a20a903d628f22ffad3e3f431d2c",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9063829787234042,
    "decisive": 235,
    "decisive_rate": 0.6300268096514745,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 91,
   "ranked": true,
   "ranks": {
    "A": 91,
    "B": 95,
    "C": 96,
    "capability": 123
   },
   "repo": "https://huggingface.co/tattrongvu/Jeffan-4B-2010",
   "scores": {
    "A": 41.06456220687438,
    "B": 36.716458621400676,
    "C": 28.517057088107215
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20658003700000335,
    "p50_s_raw": 0.02829001850000168,
    "p95_s_adjusted": 0.23990137499999947,
    "p95_s_raw": 0.04495068749999973
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN DECISION INDEX ENGINE, run unmodified: tattrongvu/Jeffan-4B-2010 at 937a008f86f4a20a903d628f22ffad3e3f431d2c, three author package files (__init__.py, engine.py, render.py), the three DI-kit base files at f8552a13b643d39f61185f5951bb5fed16bedf51 and our transport-only jeffan_shim.py. Exact ten-file code manifest sha256 f38f38d69571180283539bb9d8fad4412a6376c5d010bbe53919f8a81a5b8004, hash-checked in the container. jeffan_engine.DecisionEngine(max_len=65536, budget=65536, device=\"cuda\"), Transformers bf16, SDPA with explicit bidirectional masks for full-attention layers and causal Gated-DeltaNet layers, one read. Python 3.12.3, torch 2.13.0+cu130 (author note: 2.14.1+cu130; pyproject requires >=2.9), transformers 5.17.0, tokenizers 0.23.3, flash-linear-attention 0.5.2, fla-core 0.5.2, huggingface-hub 1.33.0. Optional causal-conv1d absent: torch fallback. vLLM is installed in the image but is not the 4b serving backend. Base image vllm/vllm-openai:v0.29.0@sha256:c2914767605584b6d8f45686b82de173ecc99e781897aa3d0a66dacd72c51ae1; derived image sha256:4f4d8a799589b6137954bd2fd7d0f2c4ac8b126ad08e5d31d261efe409bd4b51, hash-pinned wheel overlay. The submitted code was reviewed and cleared at the exact pins with the reviewed safeguards; evaluator exposure and the device were cleared for both attempts. Source review records dormant Hub-capable hf_hub_download/from_pretrained paths; the model argument is a hash-checked local directory, revision=None, offline flags set and network disabled. No author shell/subprocess, eval/exec, pickle or trust_remote_code path on the reviewed serving path. The 4b vLLM plugin was excluded and never executed. The runner and device selection change were reviewed; the amended runner was rechecked with the suite rerun and amendment recorded before retry. Weights: Apache-2.0 at this pin, public, ungated; bf16, 4,205,751,296 parameters, 8,411,558,400 weight bytes.  FIRST MEASUREMENT of Jeffan-4B-2010 (our own intake). AUTHOR'S OWN ENGINE on our rented hardware, one request at a time; our harness replaces the author's Decision Index pipeline. NATIVE OUTPUTS: next-token probabilities over the option codes, choice argmax and probabilities, noul P(true), score expectation and probabilities; the first-party transport shim adds no prompt text or probability arithmetic. AUTHOR EXPOSURE: the preregistration records training data as undisclosed (Decision Index note) and author-reported Decision Index suite figures; no overlap or contamination audit was run by us. The author's reported figures are not independently verified by us. NOT ANSWERED, counted wrong: 15 items exceed the author engine's 65,536-token max_len/budget, HTTP 422 Unsupported, exactly the pre-registered standard long positions in the pre-registered long-item set; no truncation. Answered 1,485 of 1,500 items HTTP 200; no unexpected refusal. 10 s RULE: 0 decisions over 10 s; no additional decision is counted wrong by that rule in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 0.083058 s, median 0.028291 s, p95 0.044957 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. ATTEMPT HISTORY: attempt 1, pod window 24 Oct 2026 05:52-06:21Z (about USD 1.24) on a separate disposable GPU instance, stopped the 31b row after 1,057 of 1,500 items at the runner's own time cap. The smoke under-predicted long-item latency because repeated probe prompts hit the prefix cache. The partial is retained as a receipt only, never scored or merged; the 4b row had not started. Amendment 3 corrected the first-party runner to use cold nonce-bearing probes, maximum anchor latency and a long-item budget floor; a further review preceded the retry. evaluator exposure and device ruling admitted the fresh full 31b run as the final same-use retry; 4b remained a first use. Attempt 2 is this measurement, both rows back to back on new a separate disposable GPU instance, 24 Oct 07:59-08:36Z, about USD 1.57 rental. Between attempts only our first-party runner (smoke probes and fit gate, pre-registration Amendment 3) changed; the scoring harness, model, author code, input and engine settings are unchanged, and the pinned image stack was rebuilt on the new VM. ",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5909090909090909,
       "cc": 17.107216000768457,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 50.0,
        "score": -48.678351997694634
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 17.107216000768457
      },
      "calibration_gen": {
       "accuracy": 0.37623762376237624,
       "cc": 7.562217511295402,
       "cc_by_type": {
        "choice": 30.010604453870634,
        "noul": -28.000000000000004,
        "score": 20.676048080015576
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 7.562217511295402
      },
      "composite_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 41.6822368932725,
       "cc_by_type": {
        "choice": 77.23577235772358,
        "noul": -3.703703703703709,
        "score": 51.514642025797656
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 41.6822368932725
      },
      "fairness": {
       "accuracy": 0.8333333333333334,
       "cc": 74.29181378692071,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 62.87544136076213
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 74.29181378692071
      },
      "jaggedness": {
       "accuracy": 0.49444444444444446,
       "cc": 12.959890259411873,
       "cc_by_type": {
        "choice": 48.0269489894129,
        "noul": -35.13513513513513,
        "score": 25.987856923957853
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 12.959890259411873
      },
      "llm_router": {
       "accuracy": 0.7859424920127795,
       "cc": 63.12199499130963,
       "cc_by_type": {
        "choice": 73.05626485867745,
        "noul": 67.01030927835052,
        "score": 49.29941083690091
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.12199499130963
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": -12.627343185750314,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75.0,
        "score": -13.397493474776724
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6254545454545455,
       "cc": 34.063450895608206,
       "cc_by_type": {
        "choice": 62.83012716009129,
        "noul": -2.777777777777779,
        "score": 42.138003304511116
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.063450895608206
      },
      "negation_pair_gen": {
       "accuracy": 0.6744186046511628,
       "cc": 28.994252986539696,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": -6.666666666666665,
        "score": 6.982758959619084
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 28.994252986539696
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 85.81723374265215,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 69.46371323996847
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.81723374265215
      },
      "usecase": {
       "accuracy": 0.5681818181818182,
       "cc": 30.91043554539965,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": -4.1666666666666625,
        "score": 33.662679185218536
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 30.91043554539965
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": -1.2336293925260573,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0.0,
        "score": -46.558031034721026
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 43.53012068982637,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -2.7429712638542103
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 43.53012068982637
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 62.97016399304912,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75.0,
        "score": 45.76889905879339
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 62.97016399304912
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.06279181925926,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.89414826969585
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.06279181925926
      },
      "en": {
       "accuracy": 0.6420503909643788,
       "cc": 38.65953483041661,
       "cc_by_type": {
        "choice": 65.5117706642584,
        "noul": 15.107913669064743,
        "score": 35.35892015792667
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 38.65953483041661
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.24333528280283,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 18.305862292127927
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.24333528280283
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 36.95274616427675,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 75.88556089720186
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 36.95274616427675
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 35.29290746824505,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 48.23166358120573
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 35.29290746824505
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 19.531242708314245,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50.0,
        "score": 25.353504661255577
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 19.531242708314245
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 82.54417059540985,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 47.63251178622958
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 82.54417059540985
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 13.714413463902668,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 0.0,
        "score": 12.628546189960588
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 13.714413463902668
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -13.066722136521031,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 5.244278034881356
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.81697623873661,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.11759538287652
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.81697623873661
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 5.697940383667344,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 43.06784712502799
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.697940383667344
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -39.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 29.62098153412335,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40.0,
        "score": 51.71525099255203
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 29.62098153412335
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 76.43182288942896,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 61.11632901006649
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.43182288942896
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 30.653224268712048,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 48.555417486987196
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 30.653224268712048
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 29.035136356137745,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50.0,
        "score": -19.416330062021547
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.035136356137745
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 42.508577211094895,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": -7.958139334457259
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 42.508577211094895
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 19.62220986732483,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -7.800037064692167
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.62220986732483
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7306666666666667,
       "cc": 65.21173386402343,
       "cc_by_type": {
        "choice": 65.21173386402343
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.21173386402343
      },
      "noul": {
       "accuracy": 0.568,
       "cc": 13.59999999999999,
       "cc_by_type": {
        "noul": 13.59999999999999
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 13.59999999999999
      },
      "score": {
       "accuracy": 0.5386666666666666,
       "cc": 35.71601417794966,
       "cc_by_type": {
        "score": 35.71601417794966
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.71601417794966
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.281748941189204,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": 1.1547531764323882
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 15.505884184811967,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -11.111111111111116,
        "score": 13.52938478355944
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 15.505884184811967
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 48.461591841203415,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50.0,
        "score": 44.165263328488294
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 48.461591841203415
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -23.483617154781218,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -100.0,
        "score": -11.971319300600957
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 9.948057697322461,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 21.148520918054338
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 9.948057697322461
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -35.472173838097454,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -33.333333333333336,
        "score": -110.19010641995273
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 32.3619825613593,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -14.28571428571429,
        "score": 21.17558353841963
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 32.3619825613593
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -10.059986139273484,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100.0,
        "score": 32.53781161702275
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -4.022713098007958,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60.0,
        "score": 52.0985273726428
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 39.0492321797909,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 33.33333333333333,
        "score": 22.35826042231347
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 39.0492321797909
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 28.820093796807527,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 9.1992994782779
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 28.820093796807527
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -29.197088514175146,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100.0,
        "score": 9.182928005861656
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 43.44460100858145,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 33.36643258428559
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 43.44460100858145
      },
      "model_routing": {
       "accuracy": 0.8248847926267281,
       "cc": 66.28766302911839,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 81.81818181818181,
        "score": 33.408443632809735
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.28766302911839
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 11.296360801445681,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 56.9217119628783
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 11.296360801445681
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 52.95878569800709,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 42.2561156826935
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 52.95878569800709
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": 2.8202015899512247,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50.0,
        "score": 17.573905262464507
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 2.8202015899512247
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 9.491140938992965,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 5.285017019877447
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 9.491140938992965
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 91.97693809933558,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 86.60697088163664
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 91.97693809933558
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.87939852911506,
      88.28583907929631
     ],
     "I": [
      34.57075435928627,
      44.161488911611954
     ],
     "capability": [
      59.16381168166789,
      65.51171574224394
     ],
     "composite": {
      "A": [
       28.515963494686925,
       51.612621167968996
      ],
      "B": [
       24.89561268129345,
       46.92006592201432
      ],
      "C": [
       19.802752426865915,
       35.842098033311814
      ]
     },
     "seed": 37
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.9034951152282,
     "capability": 64.00172484998512,
     "intelligence": 40.09995458474205
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7306666666666667,
      "calibration": 90.21978316183286,
      "cc": 65.21173386402343,
      "cc_by_type": {
       "choice": 65.21173386402343
      },
      "cc_tiered": 67.4905587175626,
      "cc_tiered_open": 72.15914308499305,
      "cc_tiered_sealed": 62.82197435013218,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.21173386402343
     },
     "noul": {
      "accuracy": 0.568,
      "calibration": 83.00497493367189,
      "cc": 13.59999999999999,
      "cc_by_type": {
       "noul": 13.59999999999999
      },
      "cc_tiered": 16.613900846754568,
      "cc_tiered_open": 31.68391994478951,
      "cc_tiered_sealed": 1.5438817487196292,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 13.59999999999999
     },
     "score": {
      "accuracy": 0.5386666666666666,
      "calibration": 90.48572725017985,
      "cc": 35.71601417794966,
      "cc_by_type": {
       "score": 35.71601417794966
      },
      "cc_tiered": 36.19540418990898,
      "cc_tiered_open": 39.14383250344822,
      "cc_tiered_sealed": 33.24697587636973,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.71601417794966
     }
    },
    "raw_on_A": {
     "calibration": 85.75583127024964,
     "capability": 63.842376704635804,
     "intelligence": 41.928922139021964
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 37.63127775380544,
    "Emphasis on Accuracy 60:20:20": 31.793841351745286,
    "Emphasis on Cost 20:20:60": 39.00058729810514,
    "Emphasis on Speed 20:60:20": 44.19372653768233,
    "Intelligence only": 25.792392766272524
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pragmaticfrontier (Hugging Face)",
   "axes": {
    "calibration": 83.15885665129233,
    "cost": 70.53667111174033,
    "intelligence": 25.626746881681953,
    "speed": 90.75287985569767
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09529471475409836,
      "ece_score": 80.94105704918033,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.94105704918033,
      "score_before_time_limit": 80.94105704918033,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.6595174262734584,
      "brier": 0.15627444447721173,
      "ece": 0.07921313672922253,
      "n": 373,
      "score": 84.1573726541555,
      "score_before_time_limit": 84.1573726541555,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9838709677419355,
      "interval90_width": 0.7598566308243728,
      "n": 372,
      "nrps": 0.12251524337627304,
      "score": 84.37814025054118,
      "score_before_time_limit": 84.37814025054118,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.09496097580645166
     }
    },
    "score": 83.15885665129233
   },
   "capability": 54.39280176648714,
   "composite_ci95": {
    "A": [
     9.843666545150796,
     17.645041443902656
    ],
    "B": [
     7.982146019989535,
     14.850383591606386
    ],
    "C": [
     6.835879545243608,
     12.25350100271018
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7; this base resolves identically under the earlier 1f4d1f7b...): base string google/gemma-4-E4B-it (the card's stated base_model, unverified author statement) priced as google/gemma-4-E4B-it -> deepinfra:google/gemma-4-E4B-it USD 0.02/M input, USD 0.1/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author server's own usage block: 712,537 input on the 1,485 answered items (equal to our offline count item by item, pre-registered), 0 output tokens (the model reads label probabilities at the answer position, no generation); the author count bills the state block once plus each given-order question block, not the chat template and not the reversed-order second probe, and that native field is used as reported; the 15 refused items count 0 tokens.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009500493333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009596457912457912
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jez-large-1.0.0 (pragmaticfrontier, Gemma-4-E4B label-logit decision model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.525/h), shared by the two Jez rows as two separate back-to-back uses (sealed input shredded from the node between them). The author's own server as shipped (torch backend, no vLLM serving) in a pre-built image on the stock vllm/vllm-openai:v0.30.0 base (torch 2.13.0+cu130, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2), container with --network none (cut proven from inside before the sealed input), Hub offline, every credential variable empty, read-only bundle, all 14 files of the pinned revision (15,516,502,246 bytes) sha256-verified inside the container against manifest 11092a9e... and all 169 author code files verified; our first-party loopback proxy only maps the author's over-limit 400 to 422; driven by the unchanged pinned pod_driver_v16, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 22 Oct 2026 (05:35Z start of this row) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 30.648346345077897,
    "I_sealed": 20.605147418286013,
    "base": 25.626746881681953,
    "excess": 0.20367215643617342,
    "gap": 10.043198926791884,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 63.31336811872398,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 50.92796476879522,
       "judge": 72.31377719182598,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": -47.82608695652174,
       "judge": -40.0,
       "standard": 0.0
      }
     },
     "open|score": {
      "cc": 28.63167091650972,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.0953,
       "hard": 17.99087037037035,
       "judge": 39.276420338983044,
       "standard": 26.214333333333318
      }
     },
     "sealed|choice": {
      "cc": 46.769936658102864,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 38.591418989244815,
       "judge": 41.82174338883449,
       "standard": 60.13499261759124
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -6.896551724137934,
       "hard": -54.54545454545454,
       "judge": -53.84615384615385,
       "standard": -22.388059701492537
      }
     },
     "sealed|score": {
      "cc": 15.045505596755174,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 33.295781578947384,
       "hard": 3.934931267605657,
       "judge": 16.076345833333317,
       "standard": 26.595255909090877
      }
     }
    }
   },
   "jevbench_score": 13.781856931238302,
   "key": "jez-large-1.0.0",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (HF card metadata; the card also links the Gemma 4 licence terms for the base; the HF repo has no LICENSE file; author code repo is MIT), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "pragmaticfrontier/jez-large-1.0.0@c6f511cf031cdcf8119fd67954e2fcbc6d3f2dbf",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9763779527559056,
    "decisive": 127,
    "decisive_rate": 0.34048257372654156,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 181,
   "ranked": true,
   "ranks": {
    "A": 181,
    "B": 184,
    "C": 183,
    "capability": 185
   },
   "repo": "https://huggingface.co/pragmaticfrontier/jez-large-1.0.0",
   "scores": {
    "A": 13.781856931238302,
    "B": 11.395175745424352,
    "C": 9.570733980026596
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26939871499998846,
    "p50_s_raw": 0.05969935749999422,
    "p95_s_adjusted": 0.31211633210007217,
    "p95_s_raw": 0.0810581660500361
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server pragmaticfrontier/jez at bf7c158169e5ff1db75a4b3ef3e1161cd5f50e91 (MIT; python3 -m jez.server, torch backend, --no-auth), started directly as shipped from an unchanged source copy (independent static review PASS WITH FINDINGS; no subprocess, eval/exec, pickle or trust_remote_code in the reviewed server closure; no branch on item text, task id or benchmark name). The review findings were the runtime-download fallbacks in the author code (hf_hub_download for the calibration file and from_pretrained base resolution when the checkpoint is a Hub id); they were never reached: the merged checkpoint was loaded from the local pinned /model with its own /model/calibration.json, Hub offline, container --network none, every credential variable empty; the owner cleared the code for one offline measurement on those terms. Author settings kept: order-debiased scoring (JEZ_DEBIAS=1, given and reverse order averaged), JEZ_MAX_BATCH=16, the checkpoint's calibration temperatures unchanged; no score fitting, no prompt transformation. FIRST MEASUREMENT (found by our own intake of new public Hugging Face decision models; not a free-queue submission, not in the Decision Index at pick-up). CONTEXT CAP: the author server refuses a question whose state+question exceeds 32,000 tokens and a request above 64,000 tokens (HTTP 400 max_tokens_exceeded); our reviewed loopback status proxy maps exactly that answer to 422 and passes everything else through unchanged; on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like elf-1-12b, seb-9b and the NeoHorse rows on this draw. SAME FAMILY: jez-large-1.0.0 (Gemma-4-E4B base) and jez-small-1.0.0 (Qwen3.5-0.8B base) are two checkpoints of one author measured back to back on one VM. PROVENANCE, author statements (unverified): LoRA r64/alpha128 merged into google/gemma-4-E4B-it (card); trained on 278,999 gold-label examples from 20 public sources (reports/mix-v2-manifest.json, not audited by us), no teacher model; the card reports its own acceptance gate 2 (Tier 1 vs base) as not met and states no JevBench or Decision Index result. No rule against public items was found in the reviewed code. LATENCY: maximum single decision 0.118 s on this card; no decision came near the 10 s limit. Determinism probe on the first 50 positions, two passes: 50/50 bit-identical.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 47.728766666666665,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 50.0,
        "score": 18.18629999999999
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 47.728766666666665
      },
      "calibration_gen": {
       "accuracy": 0.39603960396039606,
       "cc": -5.296255211647228,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -52.0,
        "score": 2.919293749999985
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.4158415841584158,
       "cc": -8.80096407361083,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -85.18518518518519,
        "score": 7.5627807692307485
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.5833333333333334,
       "cc": 35.68471666666667,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -46.666666666666664,
        "score": 53.72081666666667
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 35.68471666666667
      },
      "jaggedness": {
       "accuracy": 0.3055555555555556,
       "cc": -17.76067531981394,
       "cc_by_type": {
        "choice": 22.040423484119355,
        "noul": -72.97297297297297,
        "score": -2.3494764705882076
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.6389776357827476,
       "cc": 42.72053255265422,
       "cc_by_type": {
        "choice": 57.99947169146781,
        "noul": 29.89690721649485,
        "score": 40.26521874999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 42.72053255265422
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -22.07230976926854,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75.0,
        "score": 7.752142857142841
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.5490909090909091,
       "cc": 16.860791594146995,
       "cc_by_type": {
        "choice": 59.895663514835356,
        "noul": -33.333333333333336,
        "score": 24.02004460093896
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 16.860791594146995
      },
      "negation_pair_gen": {
       "accuracy": 0.29069767441860467,
       "cc": -26.800501111111107,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -100.0,
        "score": -7.068169999999996
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.7291666666666666,
       "cc": 65.37949400798935,
       "cc_by_type": {
        "choice": 65.37949400798935
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 65.37949400798935
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 52.72739511819512,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 39.99999999999999,
        "score": 48.212215384615384
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 52.72739511819512
      },
      "usecase": {
       "accuracy": 0.45454545454545453,
       "cc": 4.262127848366023,
       "cc_by_type": {
        "choice": 54.65686274509807,
        "noul": -62.5,
        "score": 20.6295208
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 4.262127848366023
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -22.651981684981692,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50.0,
        "score": -3.6702307692307867
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 18.107952222222217,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": -12.342810000000014
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 18.107952222222217
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 61.49717155316148,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75.0,
        "score": 41.34992173913044
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 61.49717155316148
      },
      "el": {
       "accuracy": 0.4,
       "cc": 8.02099897579143,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -100.0,
        "score": 57.58254999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 8.02099897579143
      },
      "en": {
       "accuracy": 0.48045178105994785,
       "cc": 8.592013966401177,
       "cc_by_type": {
        "choice": 48.15782087111456,
        "noul": -39.568345323741006,
        "score": 17.18656635182998
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 8.592013966401177
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 48.436651377027154,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75.0,
        "score": -19.902117647058827
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 48.436651377027154
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -0.8360408014571945,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 29.18586666666667
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 36.61608699690402,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 31.024731578947375
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 36.61608699690402
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 0.5619364757914317,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -50.0,
        "score": -14.794637500000007
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.5619364757914317
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.95263080206362,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 10.91712857142857
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.95263080206362
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 12.098056231577091,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": 13.482413333333342
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 12.098056231577091
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -19.5858462962963,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": 30.13134999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.06537777777777,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 76.19613333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.06537777777777
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -17.185083982683985,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": 0.39279999999999315
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 22.790673838931582,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40.0,
        "score": 31.224327906976722
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 22.790673838931582
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 23.36979110257823,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": -19.999999999999996,
        "score": 32.537187096774176
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 23.36979110257823
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -18.99189822695035,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -100.0,
        "score": 19.62005
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 15.302973188405792,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0.0,
        "score": -32.351950000000016
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 15.302973188405792
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 50.6590188172043,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 100.0,
        "score": 48.75124999999999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 50.6590188172043
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -3.979430320150655,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -27.75749999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6173333333333333,
       "cc": 50.57310702462733,
       "cc_by_type": {
        "choice": 50.57310702462733
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.57310702462733
      },
      "noul": {
       "accuracy": 0.33066666666666666,
       "cc": -33.86666666666667,
       "cc_by_type": {
        "noul": -33.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.432,
       "cc": 18.55008041516254,
       "cc_by_type": {
        "score": 18.55008041516254
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 18.55008041516254
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -30.149688235294118,
       "cc_by_type": {
        "choice": 36.0,
        "noul": -100.0,
        "score": -26.449064705882353
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -10.605294780662168,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -55.55555555555556,
        "score": 16.907373076923093
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 37.238112466124655,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 0.0,
        "score": 27.97449999999998
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 37.238112466124655
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -33.806594298245614,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": 3.8433750000000044
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": -11.968426539855066,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -100.0,
        "score": 7.572981250000021
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -0.74184968553462,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -100.0,
        "score": 35.510299999999916
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 18.85952832722833,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -71.42857142857143,
        "score": 28.007156410256407
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 18.85952832722833
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -15.346991460889763,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100.0,
        "score": 16.676795652173904
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -16.61181296296297,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60.0,
        "score": 14.331227777777766
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.18181818181818182,
       "cc": -23.690844495273936,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -66.66666666666667,
        "score": -1.6221409090908967
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 17.945389593610518,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": -1.202590909090917
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 17.945389593610518
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -57.1503943548387,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -42.41892499999997
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": -0.5887295940854571,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 32.78275555555553
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.7142857142857143,
       "cc": 53.92926226213023,
       "cc_by_type": {
        "choice": 73.11688311688313,
        "noul": 57.57575757575757,
        "score": 31.095146093749992
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 53.92926226213023
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 3.7569469609724897,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 57.3361
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.7569469609724897
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 44.48287790634003,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 16.82839230769233
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 44.48287790634003
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -11.3009231623684,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100.0,
        "score": 10.43220588235293
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -18.42371289855073,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -66.66666666666667,
        "score": 12.84480333333331
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 47.43235876880814,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 0.0,
        "score": 63.6493894736842
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 47.43235876880814
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.65436930601125,
      85.039795395276
     ],
     "I": [
      22.47448065934068,
      28.306745826814417
     ],
     "capability": [
      50.91698361027989,
      55.85159962168382
     ],
     "composite": {
      "A": [
       9.843666545150796,
       17.645041443902656
      ],
      "B": [
       7.982146019989535,
       14.850383591606386
      ],
      "C": [
       6.835879545243608,
       12.25350100271018
      ]
     },
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.15885665129233,
     "capability": 54.39280176648714,
     "intelligence": 25.626746881681953
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6173333333333333,
      "calibration": 80.94105704918033,
      "cc": 50.57310702462733,
      "cc_by_type": {
       "choice": 50.57310702462733
      },
      "cc_tiered": 55.041652388413425,
      "cc_tiered_open": 63.31336811872398,
      "cc_tiered_sealed": 46.769936658102864,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.57310702462733
     },
     "noul": {
      "accuracy": 0.33066666666666666,
      "calibration": 84.1573726541555,
      "cc": -33.86666666666667,
      "cc_by_type": {
       "noul": -33.86666666666667
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.432,
      "calibration": 84.37814025054118,
      "cc": 18.55008041516254,
      "cc_by_type": {
       "score": 18.55008041516254
      },
      "cc_tiered": 21.83858825663245,
      "cc_tiered_open": 28.63167091650972,
      "cc_tiered_sealed": 15.045505596755174,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 18.55008041516254
     }
    },
    "raw_on_A": {
     "calibration": 82.45723821371578,
     "capability": 54.02535130252623,
     "intelligence": 25.59346439133668
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 12.271949102735826,
    "Emphasis on Accuracy 60:20:20": 9.232740892685293,
    "Emphasis on Cost 20:20:60": 14.188567846789413,
    "Emphasis on Speed 20:60:20": 15.227544575204709,
    "Intelligence only": 6.731942988264025
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pragmaticfrontier (Hugging Face)",
   "axes": {
    "calibration": 81.90437649433692,
    "cost": 79.65307796501425,
    "intelligence": 4.560234849177464,
    "speed": 92.50829073456676
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09639829836065571,
      "ece_score": 80.72034032786885,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.72034032786885,
      "score_before_time_limit": 80.72034032786885,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.8927613941018767,
      "brier": 0.2317885124664879,
      "ece": 0.07312252010723858,
      "n": 373,
      "score": 85.37549597855228,
      "score_before_time_limit": 85.37549597855228,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8924731182795699,
      "interval90_width": 0.7674731182795698,
      "n": 372,
      "nrps": 0.1775305450703582,
      "score": 79.61729317658961,
      "score_before_time_limit": 79.61729317658961,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.11506179569892477
     }
    },
    "score": 81.90437649433692
   },
   "capability": 43.23230567175719,
   "composite_ci95": {
    "A": [
     0.028697570241807288,
     0.37853128896463517
    ],
    "B": [
     0.018764420428634836,
     0.26154133777550176
    ],
    "C": [
     0.019928868223477284,
     0.26286895066988547
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7; this base resolves identically under the earlier 1f4d1f7b...): base string Qwen/Qwen3.5-0.8B (the card's stated base_model, unverified author statement) priced as Qwen/Qwen3.5-0.8B -> deepinfra:Qwen/Qwen3.5-0.8B USD 0.01/M input, USD 0.05/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author server's own usage block: 707,876 input on the 1,485 answered items (equal to our offline count item by item, pre-registered), 0 output tokens (the model reads label probabilities at the answer position, no generation); the author count bills the state block once plus each given-order question block, not the chat template and not the reversed-order second probe, and that native field is used as reported; the 15 refused items count 0 tokens.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.004719173333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004766841750841751
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jez-small-1.0.0 (pragmaticfrontier, Qwen3.5-0.8B label-logit decision model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.525/h), shared by the two Jez rows as two separate back-to-back uses (sealed input shredded from the node between them). The author's own server as shipped (torch backend, no vLLM serving) in a pre-built image on the stock vllm/vllm-openai:v0.30.0 base (torch 2.13.0+cu130, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2), container with --network none (cut proven from inside before the sealed input), Hub offline, every credential variable empty, read-only bundle, all 11 files of the pinned revision (2,033,398,221 bytes) sha256-verified inside the container against manifest 26570461... and all 169 author code files verified; our first-party loopback proxy only maps the author's over-limit 400 to 422; driven by the unchanged pinned pod_driver_v16, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 22 Oct 2026 (05:46Z start of this row) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.606378088993198,
    "I_sealed": 4.51409160936173,
    "base": 4.560234849177464,
    "excess": -9.747240290724243,
    "gap": 0.09228647963146841,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 13.819134266979594,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.837837837837846,
       "hard": 1.8559295375904445,
       "judge": 20.89650626235992,
       "standard": 17.12013394725827
      }
     },
     "open|noul": {
      "cc": 0.0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -73.91304347826086,
       "judge": -90.0,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0.0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.356584848484864,
       "hard": -7.228344444444423,
       "judge": -4.739777966101699,
       "standard": -23.675696296296287
      }
     },
     "sealed|choice": {
      "cc": 13.54227482808519,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 37.10292249047015,
       "hard": 5.285069966462341,
       "judge": 15.377081292850164,
       "standard": 15.524151022990946
      }
     },
     "sealed|noul": {
      "cc": 0.0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -62.06896551724138,
       "hard": -94.54545454545455,
       "judge": -87.6923076923077,
       "standard": -85.07462686567165
      }
     },
     "sealed|score": {
      "cc": 0.0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 4.811904385964915,
       "hard": -20.03648563380276,
       "judge": -4.9763859375000274,
       "standard": -3.007532727272766
      }
     }
    }
   },
   "jevbench_score": 0.1305544579879069,
   "key": "jez-small-1.0.0",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (HF card metadata; the HF repo has no LICENSE file; author code repo is MIT), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "pragmaticfrontier/jez-small-1.0.0@4151ad0762ca95eae85b5a10c6dd6bfc10eb7627",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.85,
    "decisive": 40,
    "decisive_rate": 0.10723860589812333,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 243,
   "ranked": true,
   "ranks": {
    "A": 243,
    "B": 243,
    "C": 243,
    "capability": 216
   },
   "repo": "https://huggingface.co/pragmaticfrontier/jez-small-1.0.0",
   "scores": {
    "A": 0.1305544579879069,
    "B": 0.08771844106546485,
    "C": 0.0906628180471576
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23508084200002485,
    "p50_s_raw": 0.04254042100001243,
    "p95_s_adjusted": 0.23875566719984817,
    "p95_s_raw": 0.04437783359992409
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server pragmaticfrontier/jez at bf7c158169e5ff1db75a4b3ef3e1161cd5f50e91 (MIT; python3 -m jez.server, torch backend, --no-auth), started directly as shipped from an unchanged source copy (independent static review PASS WITH FINDINGS; no subprocess, eval/exec, pickle or trust_remote_code in the reviewed server closure; no branch on item text, task id or benchmark name). The review findings were the runtime-download fallbacks in the author code (hf_hub_download for the calibration file and from_pretrained base resolution when the checkpoint is a Hub id); they were never reached: the merged checkpoint was loaded from the local pinned /model with its own /model/calibration.json, Hub offline, container --network none, every credential variable empty; the owner cleared the code for one offline measurement on those terms. Author settings kept: order-debiased scoring (JEZ_DEBIAS=1, given and reverse order averaged), JEZ_MAX_BATCH=16, the checkpoint's calibration temperatures unchanged; no score fitting, no prompt transformation. FIRST MEASUREMENT (found by our own intake of new public Hugging Face decision models; not a free-queue submission, not in the Decision Index at pick-up). CONTEXT CAP: the author server refuses a question whose state+question exceeds 32,000 tokens and a request above 64,000 tokens (HTTP 400 max_tokens_exceeded); our reviewed loopback status proxy maps exactly that answer to 422 and passes everything else through unchanged; on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like elf-1-12b, seb-9b and the NeoHorse rows on this draw. SAME FAMILY: jez-large-1.0.0 (Gemma-4-E4B base) and jez-small-1.0.0 (Qwen3.5-0.8B base) are two checkpoints of one author measured back to back on one VM. PROVENANCE, author statements (unverified): full fine-tune of Qwen/Qwen3.5-0.8B (card); trained on 278,999 gold-label examples from 20 public sources (reports/mix-v2-manifest.json, not audited by us), no teacher model; the card reports its own acceptance gate 2 (Tier 1 vs base) as not met and states no JevBench or Decision Index result. No rule against public items was found in the reviewed code. LATENCY: maximum single decision 0.054 s on this card; no decision came near the 10 s limit. Determinism probe on the first 50 positions, two passes: 50/50 bit-identical.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.2727272727272727,
       "cc": -51.42292222222223,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": -100.0,
        "score": -62.602100000000014
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.0
      },
      "calibration_gen": {
       "accuracy": 0.2376237623762376,
       "cc": -29.50564554060622,
       "cc_by_type": {
        "choice": 1.378579003181342,
        "noul": -92.0,
        "score": 2.104484375000004
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.0
      },
      "composite_gen": {
       "accuracy": 0.25742574257425743,
       "cc": -26.079708841173883,
       "cc_by_type": {
        "choice": 8.94308943089431,
        "noul": -85.18518518518519,
        "score": -1.9970307692307676
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.0
      },
      "fairness": {
       "accuracy": 0.3125,
       "cc": -14.807080158730153,
       "cc_by_type": {
        "choice": 42.85714285714286,
        "noul": -100.0,
        "score": 12.72161666666667
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0.0
      },
      "jaggedness": {
       "accuracy": 0.18888888888888888,
       "cc": -30.711684987450216,
       "cc_by_type": {
        "choice": -3.9461020211741884,
        "noul": -100.0,
        "score": 11.811047058823531
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.0
      },
      "llm_router": {
       "accuracy": 0.3514376996805112,
       "cc": -7.7625489739989595,
       "cc_by_type": {
        "choice": 28.678348155322713,
        "noul": -46.391752577319586,
        "score": -5.574242500000004
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0.0
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -37.92644264441173,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100.0,
        "score": -39.552523809523834
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.2581818181818182,
       "cc": -31.032599796681282,
       "cc_by_type": {
        "choice": 14.900554287577448,
        "noul": -88.88888888888889,
        "score": -19.10946478873241
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0.0
      },
      "negation_pair_gen": {
       "accuracy": 0.19767441860465115,
       "cc": -37.16371999999999,
       "cc_by_type": {
        "choice": 0.0,
        "noul": -100.0,
        "score": -11.491159999999967
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.0
      },
      "option_order": {
       "accuracy": 0.3541666666666667,
       "cc": 17.443408788282294,
       "cc_by_type": {
        "choice": 17.443408788282294
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 17.443408788282294
      },
      "perturb_pair": {
       "accuracy": 0.28125,
       "cc": -36.272838284438265,
       "cc_by_type": {
        "choice": 15.91591591591592,
        "noul": -80.0,
        "score": -44.73443076923071
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0.0
      },
      "usecase": {
       "accuracy": 0.23636363636363636,
       "cc": -31.477674975163378,
       "cc_by_type": {
        "choice": 14.215686274509835,
        "noul": -100.0,
        "score": -8.648711199999969
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -18.813943345543347,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50.0,
        "score": -30.25135384615385
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.14285714285714285,
       "cc": -52.631578947368425,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": -100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0.0
      },
      "da": {
       "accuracy": 0.23076923076923078,
       "cc": -46.459182918586784,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -100.0,
        "score": -56.428239999999995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.0
      },
      "de": {
       "accuracy": 0.34375,
       "cc": -11.632148787995382,
       "cc_by_type": {
        "choice": 28.318584070796458,
        "noul": -75.0,
        "score": 11.784969565217395
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 0.30532675356921324,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 1.1022000000000087
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.30532675356921324
      },
      "en": {
       "accuracy": 0.2545612510860122,
       "cc": -25.771539752756567,
       "cc_by_type": {
        "choice": 11.692559280457841,
        "noul": -84.89208633093526,
        "score": -4.1150922077922925
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0.0
      },
      "es": {
       "accuracy": 0.41935483870967744,
       "cc": -5.414871733358922,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -25.0,
        "score": -22.72911764705883
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0.0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -24.175535883424413,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -33.333333333333336,
        "score": -8.045733333333338
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.35714285714285715,
       "cc": -4.599143962848291,
       "cc_by_type": {
        "choice": -5.882352941176468,
        "noul": 0.0,
        "score": -7.915078947368404
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0.0
      },
      "hi": {
       "accuracy": 0.14285714285714285,
       "cc": -63.525431727188085,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100.0,
        "score": -106.77741250000001
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.0
      },
      "id": {
       "accuracy": 0.0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.5333333333333333,
       "cc": 3.8031803300795417,
       "cc_by_type": {
        "choice": 57.911145752143405,
        "noul": -33.333333333333336,
        "score": -13.168271428571444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 3.8031803300795417
      },
      "ja": {
       "accuracy": 0.2,
       "cc": -36.35878715029565,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": -80.0,
        "score": -28.99693333333333
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.0
      },
      "ko": {
       "accuracy": 0.1,
       "cc": -36.63425582010583,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100.0,
        "score": -21.013878571428602
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.2857142857142857,
       "cc": -23.8965,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -100.0,
        "score": -5.022833333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "nl": {
       "accuracy": 0.1,
       "cc": -48.03832796536796,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -100.0,
        "score": -40.21888
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.38461538461538464,
       "cc": -15.141638561872801,
       "cc_by_type": {
        "choice": 46.67795175624207,
        "noul": -80.0,
        "score": -12.102867441860464
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0.0
      },
      "pt": {
       "accuracy": 0.32142857142857145,
       "cc": -19.605953278714505,
       "cc_by_type": {
        "choice": 15.144372421921032,
        "noul": -60.0,
        "score": -13.962232258064544
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": -10.591621631205669,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": -100.0,
        "score": -6.242949999999992
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.0
      },
      "tr": {
       "accuracy": 0.07692307692307693,
       "cc": -67.57375253623188,
       "cc_by_type": {
        "choice": -30.43478260869565,
        "noul": -100.0,
        "score": -72.28647500000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.2857142857142857,
       "cc": -33.33869301075269,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -35.49995000000001
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0.0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -20.91184952919021,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -33.333333333333336,
        "score": -27.70730000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.3333333333333333,
       "cc": 13.890430356493624,
       "cc_by_type": {
        "choice": 13.890430356493624
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.890430356493624
      },
      "noul": {
       "accuracy": 0.09066666666666667,
       "cc": -81.86666666666666,
       "cc_by_type": {
        "noul": -81.86666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.0
      },
      "score": {
       "accuracy": 0.2986666666666667,
       "cc": -7.835111191335686,
       "cc_by_type": {
        "score": -7.835111191335686
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.09090909090909091,
       "cc": -47.05043333333334,
       "cc_by_type": {
        "choice": -12.0,
        "noul": -100.0,
        "score": -29.151300000000013
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.20833333333333334,
       "cc": -29.85727498009237,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -100.0,
        "score": 3.595876923076935
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0.0
      },
      "customer_support": {
       "accuracy": 0.35714285714285715,
       "cc": 0.6096623306233061,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": -100.0,
        "score": 83.1298
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0.6096623306233061
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -39.28273179824562,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100.0,
        "score": -12.585037500000018
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -34.78189145531402,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -100.0,
        "score": -31.881906250000025
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.0
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -97.02319360587,
       "cc_by_type": {
        "choice": -0.62893081761006,
        "noul": -100.0,
        "score": -190.44064999999998
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": -11.012038037779222,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": -22.391856410256427
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0.0
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -38.69361545220422,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100.0,
        "score": -22.004191304347852
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.0,
       "cc": -54.962612962962965,
       "cc_by_type": {
        "choice": -25.0,
        "noul": -100.0,
        "score": -39.88783888888889
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -33.5989124813445,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -100.0,
        "score": -10.860977272727279
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.0
      },
      "lead_generation": {
       "accuracy": 0.12,
       "cc": -40.80497325581394,
       "cc_by_type": {
        "choice": -16.279069767441868,
        "noul": -100.0,
        "score": -6.135849999999965
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -28.290138844086027,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -20.354287500000012
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.37037037037037035,
       "cc": -16.352233594583083,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": -2.9914416666667165
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.0
      },
      "model_routing": {
       "accuracy": 0.42857142857142855,
       "cc": 5.444065174062047,
       "cc_by_type": {
        "choice": 40.389610389610404,
        "noul": -24.242424242424242,
        "score": 0.18500937499997816
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 5.444065174062047
      },
      "moderation_trust_safety": {
       "accuracy": 0.2222222222222222,
       "cc": -30.581622720258935,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -100.0,
        "score": -11.130664705882376
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.3333333333333333,
       "cc": -29.952590382829786,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -100.0,
        "score": -6.292311538461504
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.0
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -17.983279658070124,
       "cc_by_type": {
        "choice": 11.330049261083742,
        "noul": -50.0,
        "score": -15.279888235294115
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.17391304347826086,
       "cc": -39.641834009661835,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -100.0,
        "score": -17.476226666666683
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.25,
       "cc": -20.286962290066814,
       "cc_by_type": {
        "choice": 35.94306049822064,
        "noul": -100.0,
        "score": 3.1960526315789273
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0.0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.22516925019362,
      84.80314704701615
     ],
     "I": [
      2.6995341060300126,
      6.637224766650235
     ],
     "capability": [
      39.978396693489344,
      45.273986258801244
     ],
     "composite": {
      "A": [
       0.028697570241807288,
       0.37853128896463517
      ],
      "B": [
       0.018764420428634836,
       0.26154133777550176
      ],
      "C": [
       0.019928868223477284,
       0.26286895066988547
      ]
     },
     "seed": 40
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.90437649433692,
     "capability": 43.23230567175719,
     "intelligence": 4.560234849177464
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3333333333333333,
      "calibration": 80.72034032786885,
      "cc": 13.890430356493624,
      "cc_by_type": {
       "choice": 13.890430356493624
      },
      "cc_tiered": 13.680704547532393,
      "cc_tiered_open": 13.819134266979594,
      "cc_tiered_sealed": 13.54227482808519,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.890430356493624
     },
     "noul": {
      "accuracy": 0.09066666666666667,
      "calibration": 85.37549597855228,
      "cc": -81.86666666666666,
      "cc_by_type": {
       "noul": -81.86666666666666
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.0
     },
     "score": {
      "accuracy": 0.2986666666666667,
      "calibration": 79.61729317658961,
      "cc": -7.835111191335686,
      "cc_by_type": {
       "score": -7.835111191335686
      },
      "cc_tiered": 0.0,
      "cc_tiered_open": 0.0,
      "cc_tiered_sealed": 0.0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.0
     }
    },
    "raw_on_A": {
     "calibration": 79.01899649379038,
     "capability": 41.76596099097298,
     "intelligence": 4.512925488155576
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.10284262556076447,
    "Emphasis on Accuracy 60:20:20": 0.061053947220172385,
    "Emphasis on Cost 20:20:60": 0.15533113549209868,
    "Emphasis on Speed 20:60:20": 0.1573819915764832,
    "Intelligence only": 0.03793338673363642
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "PelaAI (Hugging Face)",
   "axes": {
    "calibration": 92.98946060661117,
    "cost": 40.92322579288957,
    "intelligence": 77.43747194470296,
    "speed": 81.4421691177115
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.022168923936636005,
      "ece_score": 95.5662152126728,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 94.29199900983717,
      "score_before_time_limit": 95.5662152126728,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.09599999999999997,
      "brier": 0.047840463085377406,
      "ece": 0.02830715981688726,
      "n": 375,
      "score": 93.8354290070939,
      "score_before_time_limit": 94.33856803662255,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9466666666666667,
      "interval90_width": 0.12422222222222223,
      "n": 375,
      "nrps": 0.042848436403401725,
      "score": 90.8409538029025,
      "score_before_time_limit": 91.57354214002268,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.06284036039807225
     }
    },
    "score": 92.98946060661117
   },
   "capability": 85.21346627565707,
   "composite_ci95": {
    "A": [
     43.56230823799144,
     44.73444334896921
    ],
    "B": [
     44.56286115961495,
     46.3345573304869
    ],
    "C": [
     43.56230823799144,
     44.73444334896921
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string google/gemma-4-31B-it (the card's base_model, in the author's capitalised spelling) priced as google/gemma-4-31b-it USD 0.09/M input, USD 0.34/M output. The exact case of 31B is required by the frozen module: it resolves to the lowercase google/gemma-4-31b-it listing, while the lowercase spelling raises 'base model lacks an explicit market-listing decision' here (the opposite of the 26B row). No explicit reference or override is used. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's server: input_tokens = SGLang prompt_tokens, including the chat template and the read-twice layout; output_tokens = 1 per question (the author's accounting for one label, not a generated explanation). Read twice is on for text-only requests of <= 3,000 read-once tokens: 1,485 items read twice, the 15 standard very long items read once. The native usage and the offline counter agree item by item with offset 0 and 0 mismatches: 2,722,505 input and 1,500 output tokens over all 1,500 answered items (byte-identical counts to the KnowLine-26B-A4B-Gen2.5 count with the same Gemma tokenizer). No refusals. The 15 standard long positions count for no row's cost on the common 1,485-item cost basis (v1.6.30); they carry 1,190,988 input and 15 output tokens, so common-basis input tokens = 1,531,517, with 1,485 output tokens. Measured arithmetic: USD 0.093159212 per 1,000 decisions on the common basis = 1.442212392x the unchanged USD 0.06459465517241379 cap; separately USD 0.163690300/1,000 over all 1,500 items = 2.534115239x. The preregistration predicted about USD 0.093 and about 1.44x on the 26B-class counts because the short items are read twice; the measurement matches. OVER COST CAP: ranked_over_cap, shown only in uncapped views and excluded from official cost-capped views; no cap exception or top-up (ruling on file).  COST ON THE COMMON BASIS: USD 0.093159 per 1,000 decisions on 1,485 items, 1.442 times the official cost cap USD 0.064595; estimated, not charged. OVER THE OFFICIAL COST CAP, DISCLOSED: ranked_over_cap; outside the official-caps views.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.16369029999999998
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.09315921212121213
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "KnowLine 31B-Gen1",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner. One VM for this row only (attempt 2; attempt 1 used a separate VM). SGLang (127.0.0.1:9080) and author front end (127.0.0.1:8000) in ONE --network none --read-only container; driver requests serial over loopback. Read-only code, model and input, blank credential variables, HF offline flags. In-container network cut proven before sealed input: only loopback, no default route, DNS and TCP fail; code inventory/hashes and model files checked; weights re-hashed against the manifest (24 files, 0 failures). SGLang /server_info sizing gate passed before sealed input (SERVER_SIZING_OK: version 0.5.21, swa_full_tokens_ratio 0.15, mem_fraction_static 0.85, dtype auto, kv_cache_dtype auto, no quantization, FP32 LM head on, max_total_num_tokens 106,983, max_req_input_len 106,977). Attempt 2 measured 24 Oct 2026 (reservation 09:49:54Z, VM created 09:51:04Z, teardown confirmed 10:25:27Z); this row's measurement container 1,401 s, run cap 5,036 s, complete=true, deadline_hit=false, rc 0; scored loop 1,252.397 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 54da5a6239bb6d60a77d5fbed9541c4ed3aded49f9e6f1961d46e1826cc9b323. Node-shred receipt verified; row cleanup verified; exact-id VM and volume teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff. ",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.65070323008781,
    "I_sealed": 76.2242406593181,
    "base": 77.43747194470296,
    "excess": -7.413064199585996,
    "gap": 2.426462570769715,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 81.5623739827074,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.2387543252595,
       "judge": 76.26895187870798,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 79.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 70.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 74.79070189113094,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 79.802915192265,
       "hard": 83.44397984101029,
       "judge": 78.30304801907488,
       "standard": 49.709520148889276
      }
     },
     "sealed|choice": {
      "cc": 85.87812548753575,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 83.34682548860874,
       "judge": 85.89618021547503,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 72.25099964369136,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 60.00000000000001,
       "judge": 75.38461538461539,
       "standard": 85.07462686567165
      }
     },
     "sealed|score": {
      "cc": 70.54359684672721,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 89.28102570375478,
       "hard": 72.52297440840516,
       "judge": 65.03461934934525,
       "standard": 65.4795935409305
      }
     }
    }
   },
   "jevbench_score": 44.376264283422806,
   "key": "knowline-31b-gen1",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 weights (model card and LICENSE); MIT author serving code (file headers); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "PelaAI/KnowLine-31B-Gen1@986803e4037f7778d24623184729a31202e3d302",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9733727810650887,
    "decisive": 338,
    "decisive_rate": 0.9061662198391421,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 80,
   "ranked": true,
   "ranks": {
    "A": 80,
    "B": 76,
    "C": 67,
    "capability": 8
   },
   "repo": "https://huggingface.co/PelaAI/KnowLine-31B-Gen1",
   "scores": {
    "A": 44.376264283422806,
    "B": 45.69727842690832,
    "C": 44.376264283422806
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6593511499998158,
    "p50_s_raw": 0.2546755749999079,
    "p95_s_adjusted": 1.088093761300297,
    "p95_s_raw": 0.4690468806501485
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: PelaAI/KnowLine-31B-Gen1 at 986803e4037f7778d24623184729a31202e3d302, knowline_server.py sha256 13ca50c4001fac0aa3899a35ff01e60f311132c571b17a8c3dc7aa6001ad2f3f (1,156 lines) and knowline_layout.py sha256 2e839fa24f1ad9c980898b2a9aa62e2154ba3f57618bd5f2fedb5756f61f062f (242 lines), plus the data file temperatures.json sha256 463a46c5ea253649bd5f43809655637d2d41939e503c81ddd01b28355d736f62 ({\"score\": 0.6944}), all mounted read-only at /model over stock SGLang 0.5.21. Only these two author Python files shipped; knowline_engine.py and serve_knowline.sh were excluded and never executed (the SGLang flags follow the author's serve_knowline.sh, plus our one pooling flag --swa-full-tokens-ratio 0.15, see below). Commands: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 -m sglang.launch_server --model-path /model --served-model-name m --tp 1 --mem-fraction-static 0.85 --enable-fp32-lm-head --swa-full-tokens-ratio 0.15 --host 127.0.0.1 --port 9080 ; env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 /model/knowline_server.py --model /model --backend sglang --url http://127.0.0.1:9080 --served-model-name knowline-31b-gen1 --temperature 1 --workers 16 --read-twice --twice-max-tokens 3000 --host 127.0.0.1 --port 8000 --temperatures /model/temperatures.json --box-prompt gemma --box-coords yxyx. Never passed: --allow-url-assets, --referenced-fields, --header, --no-read-twice, --debug-refusals, --box-calibration, --box-prompt-template (asserted on the rendered launch script; /health after the measurement shows referenced_fields false, url_assets false, no box calibration). Image: lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469, derived image sha256:579dd709d9118e745c9be62fc134fc6aac272a2ee75834da15f7cd235a025a4e (only blank-credential ENV, no package installs). Python 3.12.3, sglang 0.5.21, torch 2.13.0+cu130, transformers 5.12.1, tokenizers 0.22.2, flashinfer-python 0.6.18, sglang-kernel 0.4.7. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: a new dormant runtime download path, requests.get of http(s) image URLs (knowline_server.py _fetch, line 383), reachable only behind the command-line flag --allow-url-assets (off by default, never passed, container without network); dormant Hub-capable from_pretrained loaders in the unused --backend hf class; served tokenizer load from local /model under offline flags; HTTP client exclusively to loopback SGLang /generate (scoring and the dormant box path); local listening socket on 127.0.0.1. About 800 added lines (image, box and referenced-fields code) are not reachable from text-only requests; the one text-path change (noul criteria values rendered through render_value) leaves string, null and number criteria byte-identical and is mirrored by our offline token counter. Delta source review PASS WITH CONDITIONS (flag discipline, hash-checked snapshot, in-container offline proof, all enforced by the runner), the runner was reviewed and findings fixed. OUR ONE LAUNCH-FLAG CHANGE (pre-registration Amendment 3): attempt 1 stopped in the synthetic smoke because the default SGLang pool sizing (--swa-full-tokens-ratio 0.8) gave only 29,717 full-attention cache tokens; attempt 2 adds --swa-full-tokens-ratio 0.15, which only reallocates the same KV memory between the full-attention and sliding-window pools (live capacity max_total_num_tokens 106,983, max_req_input_len 106,977); no change to weights, BF16 precision, KV dtype, mem-fraction, context configuration or the 1,024-token attention window; bit identity against the default sizing is not promised. The amendment was reviewed and the final hashes checked (final runner sha256 18c051f1b72ec682339a11eb90025eb4c73fc4b526932c24f3faecf557fc270d), covered by the standing clearance for this amendment. Weights: Apache-2.0, public, ungated, 62,546,336,352 weight bytes in 13 BF16 shards, hash-checked file by file on the node; prereg manifest sha256 0444b046991995634646ef018d16103292eb9807147d0287898b2380b95ed98d; the shards share no LFS sha256 with the weights of the measured KnowLine 4B, 26B and 35B rows. FIRST MEASUREMENT of KnowLine-31B-Gen1 (our own intake). FIRST DENSE-GEMMA KnowLine: new weights (Gemma 4 31B-it fine-tune, LoRA rank 32 merged, then averaged 0.75 fine-tuned + 0.25 base per tensor, readout temperature 1.44 folded into the final norm and soft-cap), not a new serving version of an earlier row; the 4B, 26B and 35B KnowLine rows are different weights. AUTHOR'S OWN SERVER on our rented hardware, bf16 (no FP8; the author documents bf16 only). The server and layout files are not the bytes measured for the earlier KnowLine rows (1,156 lines against 367; see code_revision); on our text items the prompts follow the same read-twice layout. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1 for choice and noul questions and at the author's 0.6944 for score questions (temperatures.json, effective 1.0 on top of the folded 1.44; both are ranking-relevant behaviours of the author's server); choice argmax, Noul P(true), score expectation; header off, no rotation, no calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): Decision Index 0.3 public suite 70.13, self-run read twice; the card also reports JevBench / JevBench-hard evaluations; the training data is that of KnowLine-26B-A4B-Gen1 (906,122 rows, about 33% targeting Decision Index benchmarks in their request format, including train splits of public datasets rewritten in that format and synthetic items in the same format); the authors checked it against the Decision Index 0.3 rows, their own evaluation sets and public benchmarks including JevBench and removed 491 near-duplicate rows; the 0.75 averaging weight was chosen on the authors' own evaluation sets; LLM teachers labelled the synthetic tasks and not all labels were checked by a human; the card says part of the public-suite score comes from benchmarks whose tasks are in the training data. The authors say no Jev or other decision-model outputs were used as training labels. No overlap or contamination audit was run by us. The author's H20 timing (340 ms median, 3,977 ms at the 95th percentile) does not transfer to this RTX PRO 6000 measurement. CONTEXT: SGLang configuration limit 262,144 tokens, with live KV capacity max_total_num_tokens 106,983 and max_req_input_len 106,977 at our pooling flag; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 81,706 tokens. 10 s RULE: exactly 15 decisions over 10 s, the standard very long items at positions in the pre-registered long-item set, cold prefills of 76,531-81,706 tokens, up to 59.753 s each; these count wrong in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit). No decision came near the adapter's 120 s timeout. Maximum 59.753078 s, median 0.255530 s, p95 0.469112 s. DETERMINISM: first 50 positions run twice, 40/50 bit-identical, 0 argmax flips, maximum probability difference 0.0141815; the scored raw is the first pass, so scores are unaffected; bit reproducibility is not established and the cause was not investigated. ATTEMPT HISTORY: attempt 1, pod window 24 Oct 2026 08:55-09:07Z (about USD 0.49 on the runner ledger) on a separate disposable GPU instance, stopped in the synthetic smoke before any held-out item: at the default pool sizing SGLang reported 29,717 full-attention cache tokens and refused the 32,000-token probe (HTTP 422, maximum allowed input length 29,711). No sealed byte was sent and exposure stayed unspent; the attempt is retained as a receipt only, never scored or merged. pre-registration Amendment 3 changed only our SGLang launch flag (--swa-full-tokens-ratio 0.15) and added a server_info sizing gate (max_total_num_tokens >= 90,000, max_req_input_len >= 85,000) that precedes the first sealed byte; a further review and a final-hash check preceded the retry. evaluator exposure and device ruling covered attempt 1 and ruling on file admitted attempt 2 as the final attempt; evaluator exposure and device ruling was unspent at attempt 1 and is consumed by attempt 2. Attempt 2 is this measurement on new a separate disposable GPU instance, 24 Oct 09:49-10:25Z, about USD 1.54 rental. Between attempts only the pooling flag and our first-party runner and tests changed; the scoring harness, model, author code, input and all other engine settings are unchanged, and the pinned image stack was rebuilt on the new VM. OVER COST CAP: measured USD 0.093159212 per 1,000 decisions on the common basis = 1.442212392x; ranked_over_cap, uncapped views only (ruling on file). ",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1.0,
       "cc": 88.3968677123802,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 65.1906031371406
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 88.3968677123802
      },
      "calibration_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 71.16448338121529,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 52.0,
        "score": 77.40013094958434
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 71.16448338121529
      },
      "composite_gen": {
       "accuracy": 0.9405940594059405,
       "cc": 90.76705927078392,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 82.9606177400844
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.76705927078392
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 99.99941727196868,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.99825181590603
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.99941727196868
      },
      "jaggedness": {
       "accuracy": 0.8277777777777777,
       "cc": 71.83292503617992,
       "cc_by_type": {
        "choice": 76.90086621751685,
        "noul": 56.75675675675676,
        "score": 81.84115213426612
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 71.83292503617992
      },
      "llm_router": {
       "accuracy": 0.9424920127795527,
       "cc": 83.94560601208627,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 93.81443298969072,
        "score": 61.19223623966488
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.94560601208627
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 50.456047188010906,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 50.0,
        "score": 38.48154362588839
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 50.456047188010906
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 70.76021074907139,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 58.33333333333333,
        "score": 72.5322353338352
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.76021074907139
      },
      "negation_pair_gen": {
       "accuracy": 0.9302325581395349,
       "cc": 85.90181803064407,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 86.66666666666667,
        "score": 77.7054540919322
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 85.90181803064407
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 92.74806434340171,
       "cc_by_type": {
        "choice": 90.99099099099098,
        "noul": 100.0,
        "score": 87.25320203921414
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 92.74806434340171
      },
      "usecase": {
       "accuracy": 0.7954545454545454,
       "cc": 72.0418358087616,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 70.83333333333333,
        "score": 71.02746821059853
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 72.0418358087616
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 53.48166703504766,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100.0,
        "score": 17.58785824800013
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 53.48166703504766
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 71.22869587423908,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 34.42341481165732
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 71.22869587423908
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 88.06098042697631,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 88.07674659066348
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.06098042697631
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.60456618679116,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.99991829966588
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.60456618679116
      },
      "en": {
       "accuracy": 0.8809730668983493,
       "cc": 80.4997741482516,
       "cc_by_type": {
        "choice": 86.60011471668639,
        "noul": 79.85611510791367,
        "score": 75.04309262015474
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 80.4997741482516
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 72.48344614313717,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 27.238266651271214
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 72.48344614313717
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 89.07078777126578,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 99.99924855969905
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 89.07078777126578
      },
      "fr": {
       "accuracy": 1.0,
       "cc": 98.0514903309848,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 94.15447099295446
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 98.0514903309848
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 80.47847958302002,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75.0,
        "score": 66.43543874906005
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 80.47847958302002
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 63.17653605018162,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 3.5592262331637037
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.17653605018162
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 46.005914007082026,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 39.99999999999999,
        "score": 12.314803180896583
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 46.005914007082026
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 31.04667785453655,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 93.14003356360965
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 31.04667785453655
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.77688190192204,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.99731237243283
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.77688190192204
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -1.4387554084615364,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -4.316266225384613
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 75.04898636694963,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 72.7643945642429
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 75.04898636694963
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 68.3465266946212,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 60.00000000000001,
        "score": 66.25348697838334
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 68.3465266946212
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 55.426323731914415,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 57.3428009829773
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 55.426323731914415
      },
      "tr": {
       "accuracy": 1.0,
       "cc": 93.62057459269226,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 80.86172377807682
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 93.62057459269226
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.14352200896754,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 93.68863054303165
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.14352200896754
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 72.0478521212318,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 16.14355636369541
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 72.0478521212318
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.888,
       "cc": 85.53359229989094,
       "cc_by_type": {
        "choice": 85.53359229989094
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 85.53359229989094
      },
      "noul": {
       "accuracy": 0.8773333333333333,
       "cc": 75.46666666666665,
       "cc_by_type": {
        "noul": 75.46666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 75.46666666666665
      },
      "score": {
       "accuracy": 0.8186666666666667,
       "cc": 72.584235776724,
       "cc_by_type": {
        "score": 72.584235776724
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 72.584235776724
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 41.651717906240826,
       "cc_by_type": {
        "choice": 36.0,
        "noul": 50.0,
        "score": 38.95515371872248
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 41.651717906240826
      },
      "code_linting": {
       "accuracy": 1.0,
       "cc": 94.47310776421527,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 83.41932329264583
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 94.47310776421527
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 56.46158831917699,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": 1.9050901607829962
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 56.46158831917699
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -3.5233882266319547,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 16.33042011542578
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 42.42088485665628,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 75.0887415264906
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 42.42088485665628
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 36.769325748371706,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": 27.289109320586814
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 36.769325748371706
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 57.797958719568875,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 68.91208344161981
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.797958719568875
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 88.04718055411861,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 100.0,
        "score": 85.04746500730359
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 88.04718055411861
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 53.32086410018987,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 62.46259230056961
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 53.32086410018987
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 87.29303335849802,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 61.87910007549411
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 87.29303335849802
      },
      "lead_generation": {
       "accuracy": 0.88,
       "cc": 78.90939840172844,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 58.950417427407544
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 78.90939840172844
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 72.59080794067526,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 82.28855285428385
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 72.59080794067526
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 76.1961513748834,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 68.58845412465016
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 76.1961513748834
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 80.00471055982077,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 96.96969696969697,
        "score": 48.888590553921155
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.00471055982077
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 18.555142484275677,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 78.69805701136826
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 18.555142484275677
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 68.61731026200636,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 89.23168937469131
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 68.61731026200636
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 30.29288014362631,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50.0,
        "score": -14.786384199662939
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.29288014362631
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 34.430741831633014,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 46.77048636446426
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 34.430741831633014
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.99953036759018,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.99859110277058
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.99953036759018
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      89.11942711171544,
      93.23983221561662
     ],
     "I": [
      72.92480724503656,
      80.87923205534871
     ],
     "capability": [
      81.53391121608728,
      86.64467671661372
     ],
     "composite": {
      "A": [
       43.56230823799144,
       44.73444334896921
      ],
      "B": [
       44.56286115961495,
       46.3345573304869
      ],
      "C": [
       43.56230823799144,
       44.73444334896921
      ]
     },
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.98946060661117,
     "capability": 85.21346627565707,
     "intelligence": 77.43747194470296
    },
    "per_type": {
     "choice": {
      "accuracy": 0.888,
      "calibration": 94.29199900983717,
      "cc": 85.53359229989094,
      "cc_by_type": {
       "choice": 85.53359229989094
      },
      "cc_tiered": 83.72024973512157,
      "cc_tiered_open": 81.5623739827074,
      "cc_tiered_sealed": 85.87812548753575,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 85.53359229989094
     },
     "noul": {
      "accuracy": 0.8773333333333333,
      "calibration": 93.8354290070939,
      "cc": 75.46666666666665,
      "cc_by_type": {
       "noul": 75.46666666666665
      },
      "cc_tiered": 75.92501673005825,
      "cc_tiered_open": 79.59903381642512,
      "cc_tiered_sealed": 72.25099964369136,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 75.46666666666665
     },
     "score": {
      "accuracy": 0.8186666666666667,
      "calibration": 90.8409538029025,
      "cc": 72.584235776724,
      "cc_by_type": {
       "score": 72.584235776724
      },
      "cc_tiered": 72.66714936892907,
      "cc_tiered_open": 74.79070189113094,
      "cc_tiered_sealed": 70.54359684672721,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 72.584235776724
     }
    },
    "raw_on_A": {
     "calibration": 91.8267311771146,
     "capability": 84.43149101829664,
     "intelligence": 77.03625085947868
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 40.494084959728774,
    "Emphasis on Accuracy 60:20:20": 44.38930183778178,
    "Emphasis on Cost 20:20:60": 34.00416868304545,
    "Emphasis on Speed 20:60:20": 45.149204347257374,
    "Intelligence only": 51.874136134138894
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "PelaAI (Hugging Face)",
   "axes": {
    "calibration": 92.98322836048867,
    "cost": 24.72494297970482,
    "intelligence": 60.31798461023689,
    "speed": 91.72458132132627
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03408576592846662,
      "ece_score": 93.18284681430667,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 93.18284681430667,
      "score_before_time_limit": 93.18284681430667,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.2453333333333333,
      "brier": 0.08846285900576352,
      "ece": 0.03670718036761854,
      "n": 375,
      "score": 92.65856392647629,
      "score_before_time_limit": 92.65856392647629,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.34444444444444444,
      "n": 375,
      "nrps": 0.06475366111989189,
      "score": 93.10827434068305,
      "score_before_time_limit": 93.10827434068305,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.03654042603322353
     }
    },
    "score": 92.98322836048867
   },
   "capability": 76.65060648536277,
   "composite_ci95": {
    "A": [
     12.113654486902874,
     12.548317364149826
    ],
    "B": [
     12.365191710049952,
     13.07528589495351
    ],
    "C": [
     10.267901321641792,
     12.548317364149826
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7): the installed module has no exact mapping for Qwen/Qwen3.5-35B-A3B; its BASE_REFERENCES['Qwen/Qwen3.5-35B-A3B-Base'] resolves to qwen/qwen3.5-35b-a3b USD 0.3125/M input, USD 1.25/M output, applied explicitly. The exact base has a cheaper DeepInfra listing which this frozen module does not map; no pricing-module change is included in this release. The frozen reference remains USD 0.3125/M input and USD 1.25/M output, 5.00 times the official cap. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's server: input_tokens = SGLang prompt_tokens, including the chat template and the read-twice layout; output_tokens = 1 per question (the author's accounting for one label, not a generated explanation). Read twice is on for text-only requests of <= 3,000 read-once tokens: 1,485 items read twice, the 15 standard very long items read once. The native usage and the offline counter agree item by item with offset 0 and 0 mismatches: 2,708,769 input and 1,500 output tokens over all 1,500 answered items. No refusals. The 15 standard long positions count for no row's cost on the common 1,485-item cost basis (v1.6.30); common-basis input tokens = 1,528,836, with 1,485 output tokens. The pre-registration predicts about USD 0.323 per 1,000 decisions, about 5.0 times the cost cap, because the short items are read twice and the base reference is priced higher than the 4B and 26B references; measured figures are reported separately in this row. ",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.565576875
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.32297474747474747
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "KnowLine 35B-A3B-Gen1.5",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V. One VM for this row only (evaluator exposure and device ruling; --rows 35b-gen15). SGLang (127.0.0.1:9080) and author front end (127.0.0.1:8000) in ONE --network none --read-only container; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before sealed input (only loopback, no default route, DNS/TCP fail). Measured 23 Oct 2026 (reservation 08:42Z, VM created 08:43Z, teardown confirmed 08:58:23Z); measurement container 244 s (248 s including pulls), scored loop 122.727 s; run cap 2,219 s = 1.5 x the smoke-predicted 279 s plus 120 s per long item (15) under the corrected runner. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in container. Node-shred receipt verified; exact-id VM teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 65.59226733572596,
    "I_sealed": 55.043701884747826,
    "base": 60.31798461023689,
    "excess": 0.7090386806224274,
    "gap": 10.548565450978138,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 78.95373257854448,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 75.46398238439761,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 55.547273982056595,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 62.275795446576836,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.61724391609044,
       "hard": 64.49987045709548,
       "judge": 55.519436791323265,
       "standard": 60.791459173663085
      }
     },
     "sealed|choice": {
      "cc": 74.63518188960879,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 88.56416772554002,
       "hard": 70.85694460506532,
       "judge": 72.67384916748287,
       "standard": 78.16916262391901
      }
     },
     "sealed|noul": {
      "cc": 37.1236894860158,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 32.72727272727274,
       "judge": 35.384615384615394,
       "standard": 34.32835820895524
      }
     },
     "sealed|score": {
      "cc": 53.37223427861889,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.89572537236607,
       "hard": 49.955408088998645,
       "judge": 48.9797607985226,
       "standard": 59.532851331130246
      }
     }
    }
   },
   "jevbench_score": 12.431479016136572,
   "key": "knowline-35b-a3b-gen1.5",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 weights (model card and LICENSE); MIT author serving code (file headers); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "PelaAI/KnowLine-35B-A3B-Gen1.5@9adcf717361b2f7d2946d093f1e85efdf1e59010",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.950530035335689,
    "decisive": 283,
    "decisive_rate": 0.7546666666666667,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 185,
   "ranked": true,
   "ranks": {
    "A": 185,
    "B": 178,
    "C": 166,
    "capability": 51
   },
   "repo": "https://huggingface.co/PelaAI/KnowLine-35B-A3B-Gen1.5",
   "scores": {
    "A": 12.431479016136572,
    "B": 12.834904478641016,
    "C": 12.431479016136572
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2451744479999661,
    "p50_s_raw": 0.04758722399998305,
    "p95_s_adjusted": 0.2741995019998285,
    "p95_s_raw": 0.06209975099991425
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: PelaAI/KnowLine-35B-A3B-Gen1.5 at 9adcf717361b2f7d2946d093f1e85efdf1e59010, knowline_server.py sha256 ad922e7d2d717a6b9b936983ff5e5d69534edc593ed54b58732dc8ff60baa67e and knowline_layout.py sha256 aa0fb35619f4883edd6c3bd1cc6dc259f41f082cfe0471e586c60d783da2af9d, both mounted read-only at /model over stock SGLang 0.5.21. Only these two author Python files shipped; knowline_engine.py and serve_knowline.sh were excluded and never executed. Commands: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 -m sglang.launch_server --model-path /model --served-model-name m --tp 1 --quantization fp8 --mem-fraction-static 0.72 --mamba-radix-cache-strategy extra_buffer --enable-fp32-lm-head --host 127.0.0.1 --port 9080 ; env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 /model/knowline_server.py --model /model --backend sglang --url http://127.0.0.1:9080 --served-model-name knowline-35b-gen15 --temperature 1 --workers 16 --read-twice --twice-max-tokens 3000 --host 127.0.0.1 --port 8000. Image: lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469, derived image sha256:a829b8992c579540f1231a66ddd7732a470a22379d85a9f2138b6eb58bc1c187 (only blank-credential ENV, no package installs). Python 3.12.3, sglang 0.5.21, torch 2.13.0+cu130, transformers 5.12.1, tokenizers 0.22.2, flashinfer-python 0.6.18, sglang-kernel 0.4.7. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: dormant Hub-capable from_pretrained loaders in the unused --backend hf class; served tokenizer load from local /model under offline flags; HTTP client exclusively to loopback SGLang /generate. The source and runner were reviewed, with findings resolved in the amendments; the runner cap fix of Amendment 1c was independently reviewed (PASS) before this pod. Weights: Apache-2.0, public, ungated, 71,903,879,336 weight bytes, hash-checked file by file; pre-registration manifest sha256 61c753a9be64d055d16d69f2efa10b4346331844a447ed9799cd41357e38adab; byte-identical to Gen1 weights (revision e800968d). FIRST MEASUREMENT of this .5 serving version (our own intake). SAME WEIGHTS AS Gen1 (no earlier-generation KnowLine row exists on the board): the shards, tokenizer, configuration and chat template are byte-identical to the predecessor; the new author layout reads short text inputs twice, so this measures a new serving version, not new weights. AUTHOR'S OWN SERVER on our rented hardware, online FP8. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1; choice argmax, Noul P(true), score expectation; header off, no rotation or calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): the authors adopted read twice after checking their own evaluation sets and a fresh full run of the Decision Index 0.3 public suite; the card also reports JevBench / JevBench-hard evaluations. Training-data, Decision Index format-data and decontamination disclosures are inherited from Gen1; no overlap or contamination audit was run by us. The authors say no Jev or other decision-model outputs were used as training labels. The author's H20 timing does not transfer to this RTX PRO 6000 measurement. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 3.397 s (median 0.0476 s, p95 0.0621 s); the slowest decisions are the standard very long items of about 80,000 tokens. DETERMINISM: first 50 positions run twice, 2/50 bit-identical, 1 argmax flip, maximum probability difference 0.0395585 (the pre-registration expected 0 flips: a prediction miss; the scored raw is the first pass, so scores are unaffected; the 4B and 26B rows had 0 flips); bit reproducibility is not established and the cause was not investigated. CONTEXT: SGLang configuration limit 262,144 tokens; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 80,522 tokens. SEPARATE POD: this row was measured alone on its own pod after the first pod (4B and 26B rows) ended with the 26B row flagged partial by the runner (its data are complete); the 35B exposure use was unspent at that point, the runner cap was corrected (Amendment 1c) and independently reviewed, and the run completed with complete=true, no deadline hit and rc 0 (run cap 2,219 s, scored loop 122.7 s). OVER COST CAP: read-twice with this base reference is predicted about 5.0 times the cap on the common basis; measured USD 0.322975 per 1,000 decisions on the common basis = 5.00x the cap (all rows 0.565577 = 8.76x), ranked_over_cap. ",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 78.53724442653552,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 43.94506661293989
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 78.53724442653552
      },
      "calibration_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 37.12965075982285,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 19.999999999999996,
        "score": 42.290330858471734
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 37.12965075982285
      },
      "composite_gen": {
       "accuracy": 0.8712871287128713,
       "cc": 78.59353672698349,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 70.37037037037037,
        "score": 71.9143048512305
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 78.59353672698349
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 95.05631152521637,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 85.1689345756491
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 95.05631152521637
      },
      "jaggedness": {
       "accuracy": 0.5944444444444444,
       "cc": 32.52691423486501,
       "cc_by_type": {
        "choice": 56.689124157844084,
        "noul": -8.108108108108103,
        "score": 48.99972665485905
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 32.52691423486501
      },
      "llm_router": {
       "accuracy": 0.865814696485623,
       "cc": 75.68588856509247,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 85.56701030927836,
        "score": 59.717299746305486
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 75.68588856509247
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -34.50220911933348,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": -100.0,
        "score": -26.08727251929075
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 27.769786201064957,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -25.0,
        "score": 57.79389468566909
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 27.769786201064957
      },
      "multilingual": {
       "accuracy": 0.7236363636363636,
       "cc": 49.53498435009352,
       "cc_by_type": {
        "choice": 72.61167264427779,
        "noul": 16.666666666666675,
        "score": 59.32661373933609
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 49.53498435009352
      },
      "negation_pair_gen": {
       "accuracy": 0.7906976744186046,
       "cc": 55.270855701977574,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 26.66666666666666,
        "score": 45.81256710593274
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 55.270855701977574
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 84.2589451817055,
       "cc_by_type": {
        "choice": 90.99099099099098,
        "noul": 100.0,
        "score": 61.785844554125546
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.2589451817055
      },
      "usecase": {
       "accuracy": 0.7090909090909091,
       "cc": 51.4472003671922,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 29.166666666666675,
        "score": 50.91022855255696
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 51.4472003671922
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 43.81520765690724,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": 38.58848011357887
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 43.81520765690724
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 25.42097445128751,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 30.33358387613595
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 25.42097445128751
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 92.14511885715005,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 92.36456011127314
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 92.14511885715005
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 44.1730779469375,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": 0.0,
        "score": 99.55833998606387
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.1730779469375
      },
      "en": {
       "accuracy": 0.7654213727193745,
       "cc": 61.10073141265881,
       "cc_by_type": {
        "choice": 77.59363444429528,
        "noul": 49.64028776978417,
        "score": 56.06827202389697
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 61.10073141265881
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 71.25620747667905,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 23.556550651896877
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.25620747667905
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 53.099368633050915,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 91.53854305762268
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 53.099368633050915
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 80.39360734662309,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 83.53376321633982
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 80.39360734662309
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 41.58116161609464,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0.0,
        "score": 41.50326138459677
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 41.58116161609464
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 75.32642366956557,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 40.00888909131558
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 75.32642366956557
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 14.49636466714197,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0.0,
        "score": 0.6773386400279802
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 14.49636466714197
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -1.8225924303067036,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 38.976667153524346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.22713475233114,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 98.34807092366006
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.22713475233114
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 3.7864891493847828,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 11.359467448154348
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 3.7864891493847828
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 50.02576937954863,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0.0,
        "score": 65.31217906543388
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 50.02576937954863
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 59.34634800662246,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 60.46685780890688
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 59.34634800662246
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 40.91896828063803,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 79.35264952276515
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.91896828063803
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 47.842968553972014,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": -6.471094338083971
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 47.842968553972014
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.03295672336854,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 83.6149992023637
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.03295672336854
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 49.64993222925987,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 15.616463354446298
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 49.64993222925987
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8146666666666667,
       "cc": 76.06153963910522,
       "cc_by_type": {
        "choice": 76.06153963910522
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 76.06153963910522
      },
      "noul": {
       "accuracy": 0.7173333333333334,
       "cc": 43.466666666666676,
       "cc_by_type": {
        "noul": 43.466666666666676
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 43.466666666666676
      },
      "score": {
       "accuracy": 0.68,
       "cc": 56.30595230792736,
       "cc_by_type": {
        "score": 56.30595230792736
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 56.30595230792736
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 14.797508704824592,
       "cc_by_type": {
        "choice": 52.0,
        "noul": 0.0,
        "score": -7.607473885526228
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 14.797508704824592
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 58.11375223901799,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 55.55555555555556,
        "score": 37.419241534169224
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 58.11375223901799
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 35.666240626404736,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50.0,
        "score": -26.741115519159784
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 35.666240626404736
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -27.686685872344828,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -12.884619020543253
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 62.73448675841547,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 83.8556341882899
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 62.73448675841547
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -15.37956527962207,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 0.0,
        "score": -83.24561407785993
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 18.904770744015718,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -42.85714285714286,
        "score": 38.78714136369983
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 18.904770744015718
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 47.56112704589649,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 0.0,
        "score": 63.58930448263722
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 47.56112704589649
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 40.756925480947764,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 83.93744310950997
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 40.756925480947764
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 69.18952787494685,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 53.74988269693192
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 69.18952787494685
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 43.90782621530675,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 11.111111111111116,
        "score": 20.612367534809117
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.90782621530675
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -27.40129750584028,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -19.838301119671375
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 66.25297917029879,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 50.27525229016698
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 66.25297917029879
      },
      "model_routing": {
       "accuracy": 0.8525345622119815,
       "cc": 72.27971344822836,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 84.84848484848484,
        "score": 48.35429185983661
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 72.27971344822836
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 13.455828237937178,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 74.91642905162344
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 13.455828237937178
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 36.11315723953092,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": -33.333333333333336,
        "score": 58.385896973931665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 36.11315723953092
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 40.47435635108647,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": 50.97971929956482
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 40.47435635108647
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 13.82491470436007,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 66.1124252725005
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 13.82491470436007
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.78505156447903,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.35515469343709
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.78505156447903
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.00685715125017,
      92.14901128901427
     ],
     "I": [
      55.14812372594048,
      64.35103732240668
     ],
     "capability": [
      71.82312622073084,
      77.47359939236085
     ],
     "composite": {
      "A": [
       12.113654486902874,
       12.548317364149826
      ],
      "B": [
       12.365191710049952,
       13.07528589495351
      ],
      "C": [
       10.267901321641792,
       12.548317364149826
      ]
     },
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.98322836048867,
     "capability": 76.65060648536277,
     "intelligence": 60.31798461023689
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8146666666666667,
      "calibration": 93.18284681430667,
      "cc": 76.06153963910522,
      "cc_by_type": {
       "choice": 76.06153963910522
      },
      "cc_tiered": 76.79445723407663,
      "cc_tiered_open": 78.95373257854448,
      "cc_tiered_sealed": 74.63518188960879,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.06153963910522
     },
     "noul": {
      "accuracy": 0.7173333333333334,
      "calibration": 92.65856392647629,
      "cc": 43.466666666666676,
      "cc_by_type": {
       "noul": 43.466666666666676
      },
      "cc_tiered": 46.33548173403619,
      "cc_tiered_open": 55.547273982056595,
      "cc_tiered_sealed": 37.1236894860158,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.466666666666676
     },
     "score": {
      "accuracy": 0.68,
      "calibration": 93.10827434068305,
      "cc": 56.30595230792736,
      "cc_by_type": {
       "score": 56.30595230792736
      },
      "cc_tiered": 57.82401486259786,
      "cc_tiered_open": 62.275795446576836,
      "cc_tiered_sealed": 53.37223427861889,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.30595230792736
     }
    },
    "raw_on_A": {
     "calibration": 89.49061199462261,
     "capability": 75.80512414699561,
     "intelligence": 62.11963629936862
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 10.799803296481864,
    "Emphasis on Accuracy 60:20:20": 12.095390476444106,
    "Emphasis on Cost 20:20:60": 8.215822744700574,
    "Emphasis on Speed 20:60:20": 13.62576809910545,
    "Intelligence only": 14.749503825986924
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "FrontiersMind (FrontiersMind on Hugging Face)",
   "axes": {
    "calibration": 89.11821147102742,
    "cost": 17.92472424211776,
    "intelligence": 55.45890006642501,
    "speed": 85.46346745302206
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04496852459016423,
      "ece_score": 91.00629508196715,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.00629508196715,
      "score_before_time_limit": 91.00629508196715,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.26809651474530827,
      "brier": 0.105628298230563,
      "ece": 0.07878981233243955,
      "n": 373,
      "score": 84.24203753351209,
      "score_before_time_limit": 84.24203753351209,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9543010752688172,
      "interval90_width": 0.31496415770609315,
      "n": 372,
      "nrps": 0.07062718985439076,
      "score": 92.10630179760304,
      "score_before_time_limit": 92.10630179760304,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.043623387096774276
     }
    },
    "score": 89.11821147102742
   },
   "capability": 72.28855576872621,
   "composite_ci95": {
    "A": [
     5.197359394601136,
     5.376101341058595
    ],
    "B": [
     5.41156944489018,
     5.712684941761912
    ],
    "C": [
     3.656661809888081,
     5.28767825747465
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the rates for this base string are identical in the earlier module 81e3578f): base string Qwen/Qwen3.8-27B (the card's and decision_config.json's base_model) priced as qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's own server: input_tokens = tokens attended for the one question, incl. system prompt, state, options and chat template (readout.py:177); output_tokens = the token count of the JSON text of the returned answers (readout.py:202), a synthetic number because nothing is generated, so it depends on the answer (109 per answered item on average here) and is billed at the output rate, which is most of this row's cost. Our offline count through a first-party transcription of the author's prompt rendering and the pinned tokenizer predicted the server's INPUT numbers EXACTLY, item by item: 763,765 input tokens over the 1,485 answered items, and 15 HTTP 422 refusals at the pre-registered ids and positions (the 15 refused items count 0 tokens). OUTPUT tokens: the server reported 162,504 against our pre-registered planning band of 143,966 to 150,660; the server counted 1 to 18 more tokens than our model of the answer-JSON length on 1,354 of the 1,485 answered items. That prediction miss is ours (our output-length model was wrong), is disclosed here, and changes no score: the cost uses the server's native usage like every other row. OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.544305 per 1,000 decisions on the common basis (USD 0.538862 over all 1,500 items) is 8.426x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them); their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5388622000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.5443052525252525
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "FrontiersMind Lumma-fev-27B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB, driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h, 300 GB disk, host RAM 90,484 MiB. The author's own `python3 -m lumma_fev.server` on 127.0.0.1:8000 inside a docker container with network none (cut proven from inside before any item hash or model use: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, 16 GB shared memory, read-only weights and code, only /output writable, empty credential variables (names-only check), HF offline flags present in the container configuration (receipt asserted before the cut proof), one request at a time, timed over loopback on the VM. Measured 22 Oct 2026 (VM created 10:31Z; sealed run 10:36:58Z to 10:41:32Z, scored loop 234 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked on the node and inside the container). One pass: 1,500 requests plus a 50-item determinism repeat and a health check. The input was shredded from the node and its absence verified; the VM and its volume were removed by exact id (confirmed 10:42:01Z). Estimated rental cost USD 0.49 for this VM plus USD 0.34 for the earlier first VM that failed before any sealed byte. A first VM (10:19Z to 10:27Z) stopped at the offline-flags proof before any sealed byte was sent and was deleted (see code_revision).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 61.03467151530754,
    "I_sealed": 49.88312861754248,
    "base": 55.45890006642501,
    "excess": 1.3120161274093434,
    "gap": 11.151542897765054,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 78.56754551019564,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 79.2387543252595,
       "judge": 80.22412656558998,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 42.275362318840585,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 62.26110671688638,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.7,
       "hard": 60.33759259259259,
       "judge": 70.05949152542374,
       "standard": 52.69111111111111
      }
     },
     "sealed|choice": {
      "cc": 73.84071346202205,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 69.81612119810339,
       "judge": 72.67384916748287,
       "standard": 74.3724952541658
      }
     },
     "sealed|noul": {
      "cc": 27.520998815903607,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 55.172413793103445,
       "hard": 29.0909090909091,
       "judge": 7.692307692307687,
       "standard": 40.29850746268657
      }
     },
     "sealed|score": {
      "cc": 48.2876735747018,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 71.34868421052632,
       "hard": 40.68315492957747,
       "judge": 47.96874999999999,
       "standard": 52.44459090909089
      }
     }
    }
   },
   "jevbench_score": 5.313846759444804,
   "key": "lumma-fev-27b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card metadata, full LICENSE and NOTICE files; serving package lumma-fev 0.2.0 is Apache-2.0 AND MIT; base Qwen/Qwen3.8-27B is Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "FrontiersMind/Lumma-fev-27B@2c080fb4b44bbad2a140353393d935e8b7af8bcb",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9267399267399268,
    "decisive": 273,
    "decisive_rate": 0.7319034852546917,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 206,
   "ranked": true,
   "ranks": {
    "A": 206,
    "B": 205,
    "C": 205,
    "capability": 73
   },
   "repo": "https://huggingface.co/FrontiersMind/Lumma-fev-27B",
   "scores": {
    "A": 5.313846759444804,
    "B": 5.598776082292363,
    "C": 4.539928665509053
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.43387249599995814,
    "p50_s_raw": 0.14193624799997906,
    "p95_s_adjusted": 0.6550751642999671,
    "p95_s_raw": 0.25253758214998356
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own serving package, unmodified: PyPI `lumma-fev` 0.2.0 (wheel sha256 ac8e5f9380b3ea3591ae7b2eb075b2ca28d2d362038267548636e91352b19a91 = the hash PyPI publishes; sdist sha256 60a9b79cd91d7e3fbdaf18c9a16573e1273dd9959eb2c1639e9c6a95401e8c87; five Python files, 445 lines, byte-identical between wheel and sdist; uploaded 14 Oct 2026). The Hub repository itself ships no server code (the AutoJev source folder was removed by the authors on 14 Oct). No shim, no status-mapping proxy and no prompt, answer or usage rewrite was used: the author's server speaks the frozen POST /v1/systemone wire and returns usage natively, the refusals are its own HTTP 422, and the frozen TypeSafeAdapter, our first-party pod runner and pod driver were unchanged. Installed on the rented pod only, from local files, `pip install --no-deps --no-index --force-reinstall` of exactly five hash-checked wheels (lumma_fev-0.2.0, transformers-5.17.0 sha256 78ec1ce21579b38dfb83950a0658cd119f87212a2fcfdff478096ce9d6c03801, huggingface_hub-1.33.0 04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899, fastapi-0.136.3 3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620, uvicorn-0.54.0 505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf) inside a `docker build --network none --pull=false`; nothing is pulled from any index. Base image: stock vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90, used only as a CUDA / PyTorch base (torch 2.13.0+cu130, Python 3.12.3, pydantic 2.13.5, httpx 0.28.1, safetensors 0.8.0, tokenizers 0.23.2); vLLM itself is NOT used, the author serves with plain transformers and PyTorch. Derived image sha256:79d8477ca43c484d5f13bd15d85b44b5c4acd3df7b72281e07ab37d794ec62b6. In the container, before any weights, every installed lumma_fev/*.py was compared to the pinned PyPI file, the exact versions above were asserted, and flash-linear-attention, causal-conv1d and the Hub `kernels` package were asserted NOT importable. Started as `python3 -m lumma_fev.server --model /models/lumma-fev-27b/snapshots/2c080fb4b44bbad2a140353393d935e8b7af8bcb --host 127.0.0.1 --port 8000 --name lumma-fev-27b` with HF_HUB_OFFLINE=1, TRANSFORMERS_OFFLINE=1, USE_HUB_KERNELS=NO and an empty LUMMA_FEV_API_KEY (no --cors, no --api-key, no --device-map). As shipped: bfloat16, sdpa attention, readout temperature 0.93705 (decision_config.json), context limit 8,192 tokens (readout.py:11, not configurable from the command line), one request at a time (the server holds a lock). The optional speed kernels (flash-linear-attention, causal-conv1d) were NOT installed, so transformers' slower PyTorch fallback ran for the gated-delta-net layers; they are unreviewed extra code and a kernel variant would be a separate run. Code gate: cleared by the benchmark owner (22 Oct 2026 10:11Z) on a written list of the disclosed code patterns below. DISCLOSED CODE PATTERNS, none reachable in our configuration: (1) Hub downloads of decision_config.json, readout.safetensors or an image processor (readout.py:22-24, 129-131, 154-155) fire only when --model is not a local folder or a request carries images; we pass a local hash-verified folder and the frozen adapter sends no images; (2) `trust_remote_code=True` (local.py:43) in a fallback reached only when decision_config.json is absent; it is present (sha256 e212eb8cf3eb3759c703a19e0b565e4e06d55e4be96caf96d14b0d3c8de30427, asserted in the container before the cut proof) and the snapshot has no .py file and no auto_map; (3) transformers' optional Hub-kernel loaders in the Qwen3.5 model code (modeling_qwen3_5.py:217, 249-300, 437, 500, 840), inactive because USE_HUB_KERNELS=NO and the `kernels` package is not installed; (4) one environment read (LUMMA_FEV_API_KEY, left empty) and an imported but never used httpx client class (client.py). The authors' code has no subprocess, socket, eval, exec, pickle or torch.load call and no request-body logging. The source review (PASS) and the independent runner review both found nothing that required an owner gate; we asked for the owner's clearance anyway under our foreign-code rule. Reviews: a static source review (PASS), an independent runner review and a delta review (one must: a receipt proving the offline flags inside the scored container before the cut proof, closed and re-checked by us before launch); the sealed-input handling and the rented device were admitted with conditions, and the conditions were met. A first VM (before any sealed byte existed) failed exactly that offline-flags proof, because the flags reached the scored container only through its `env -i` entrypoint and not through the Docker container configuration; we then added the same three flags as `-e` options on `docker run` for the scored container (a one-line runner change checked by us with the static checks and the offline proof, fit and guard tests, not by a further independent review round) and the measurement ran once on a second VM; the first VM was deleted. Weights: FrontiersMind/Lumma-fev-27B at 2c080fb4b44bbad2a140353393d935e8b7af8bcb (weights, config, decision_config.json, readout.safetensors, tokenizer and chat template unchanged since the first upload on 14 Oct 2026; later commits only README, NOTICE and benchmark images), 23 files (11 bfloat16 shards, readout.safetensors, tokenizer.json, 10 small files; 52,193,544,229 bytes; manifest sha256 8411f98f33617fea738d6b6ecb932f6c45bbd214391d3615d4043a5e7d09d590), fetched by us from the commit-pinned Hub revision on the pod before the network was cut, and sha256-verified file by file (shard sha256 = the Hub LFS oid, small files = git blob oid; 'VERIFY OK 23 files') on the host before the smoke; the three benchmark images were not used. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry; no outside request for this model and no earlier JevBench measurement). AUTHOR-DEFAULT CONTEXT LIMIT: the author's server accepts at most 8,192 tokens per question including state, options and chat template and never truncates; above that it answers HTTP 422 itself ('a question needs N tokens with its state; the limit is 8192 and input is never truncated'). The 15 items of 76,368-80,526 tokens were refused exactly as pre-registered by id and position and are scored wrong, like the other self-hosted rows on this draw; the largest answered item is 1,864 tokens, nowhere near the limit. No status proxy was used. SERVER IS THE AUTHOR'S OWN: lumma_fev.server from the PyPI package, not the Hub repository (which ships no server code); see code_revision for the exact bytes, the disclosed code patterns and the reviews. SCORE QUESTIONS ARE OUTSIDE THE TRAINED TYPES: the card states the model was trained on noul and choice questions and that score questions are accepted but untested; about a quarter of this draw's items are score questions, answered natively by the same readout (probabilities per level), with nothing added or rewritten by us. OPTIONAL KERNELS OFF: flash-linear-attention and causal-conv1d (recommended by the card for speed) were not installed, so transformers' slower PyTorch fallback ran; kernel variants can change speed and numerical results; numerical parity against an optional-kernel configuration was not tested. SERIAL ONLY: the server handles one request at a time and our driver sent one at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips, max probability difference 0.0). TRAINING DATA: the card gives only 'about 100K samples' in English, Hindi, Tamil and Kannada and names no datasets; there is no training-data licence table and no contamination or overlap statement in the pinned card (the authors' blog post linked from the card was not read). The Decision Index 0.4.1 row carries no overlap or contamination flag. We ran no overlap check, and no JevBench use or denial is declared, so whether its training data overlaps JevBench sources is unknown to us. LICENCE: Apache-2.0 in the card metadata with a full LICENSE and a NOTICE file; the serving package is Apache-2.0 AND MIT (its prompt and scoring follow AutoJev, MIT). SELF-REPORTED, unverified: the card's three benchmark charts (accuracy against 'TypeSafe Jev 1.13.0' on public sets; not transcribed or checked here) and no JevBench or Decision Index number is claimed by the authors. The Decision Index 0.4.1 (22 Oct 2026 snapshot) states for this model: full (balanced_skill) score 59.97, public 57.15 (its kit rescore equals the submitted claim), same-skills 56.59, new-domains 51.01, decision agreement with the submitted rows 99.25% on 825 rows against a 98.5% bar (private tests run by the lab on one RTX PRO 6000 with the author's serving), latency median 100 ms and p95 1,075 ms, and that questions over its declared 8,192-token limit count as unsupported. 10 s RULE: max measured decision latency 0.48 s over all 1,500 rows (answered items: median 0.14 s, p95 0.25 s); 0 decisions over 10 s. In the synthetic smoke before the run a 8,000-token request took 2.17 s and an exact 8,192-token request 2.22 s, but no answered item is larger than 1,864 tokens. OUTPUT TOKENS: our pre-registered output-token planning band was missed (see cost_basis); every other pre-registered prediction (refusal ids and positions, answered count, input tokens item by item, determinism) was met.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 70.49,
       "cc_by_type": {
        "choice": 75.0,
        "noul": 100.0,
        "score": 36.47
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 70.49
      },
      "calibration_gen": {
       "accuracy": 0.6039603960396039,
       "cc": 37.87697574231177,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 4.0000000000000036,
        "score": 44.625625
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 37.87697574231177
      },
      "composite_gen": {
       "accuracy": 0.801980198019802,
       "cc": 58.0487336993028,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 25.92592592592593,
        "score": 51.47230769230768
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 58.0487336993028
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.50870370370372,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 90.52611111111112
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.50870370370372
      },
      "jaggedness": {
       "accuracy": 0.5277777777777778,
       "cc": 18.788328277890084,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -29.729729729729726,
        "score": 43.84254901960786
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 18.788328277890084
      },
      "llm_router": {
       "accuracy": 0.8945686900958466,
       "cc": 79.20319390999602,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 85.56701030927836,
        "score": 61.55212499999998
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 79.20319390999602
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -14.169947635411548,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -25.0,
        "score": -30.91190476190475
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.7127272727272728,
       "cc": 51.028387885306735,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": 19.444444444444443,
        "score": 60.05089201877934
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 51.028387885306735
      },
      "negation_pair_gen": {
       "accuracy": 0.7441860465116279,
       "cc": 42.64922222222223,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 6.666666666666665,
        "score": 21.281000000000017
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 42.64922222222223
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 76.1606853006853,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 80.0,
        "score": 69.50307692307692
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.1606853006853
      },
      "usecase": {
       "accuracy": 0.7272727272727273,
       "cc": 54.57329856209151,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 37.5,
        "score": 53.180680000000024
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.57329856209151
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 1.5839194139193964,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -88.10538461538466
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 1.5839194139193964
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 35.98355555555555,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 41.284
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 35.98355555555555
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 76.09684750545081,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 77.18434782608696
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 76.09684750545081
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.48848230912478,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 99.98500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.48848230912478
      },
      "en": {
       "accuracy": 0.7523892267593397,
       "cc": 55.34845437949974,
       "cc_by_type": {
        "choice": 76.27561294101854,
        "noul": 36.690647482014384,
        "score": 53.0791027154663
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 55.34845437949974
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 63.926572945654605,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 1.5676470588235292
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 63.926572945654605
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.65326047358833,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.41333333333334
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.65326047358833
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 82.40638802889576,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 89.57210526315788
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 82.40638802889576
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 33.07397346368713,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 25.0,
        "score": 24.501249999999942
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 33.07397346368713
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 61.90822254388893,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -0.24571428571433795
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 61.90822254388893
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 26.341737004677427,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0.0,
        "score": 7.619333333333311
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 26.341737004677427
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 1.1280423280423264,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 47.82857142857143
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 1.1280423280423264
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.97777777777777,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.93333333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.97777777777777
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -0.30134199134198525,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 25.070000000000014
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 51.123878101349284,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 0.0,
        "score": 60.98906976744185
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 51.123878101349284
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 69.44292705026581,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 61.36354838709678
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 69.44292705026581
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 45.32205673758866,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 67.03000000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 45.32205673758866
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 44.2522463768116,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50.0,
        "score": 26.23500000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 44.2522463768116
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.56564516129032,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 61.955000000000005
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.56564516129032
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 30.91888888888889,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 26.090000000000014
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 30.91888888888889
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8093333333333333,
       "cc": 75.37266308195717,
       "cc_by_type": {
        "choice": 75.37266308195717
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.37266308195717
      },
      "noul": {
       "accuracy": 0.6746666666666666,
       "cc": 34.93333333333333,
       "cc_by_type": {
        "noul": 34.93333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 34.93333333333333
      },
      "score": {
       "accuracy": 0.672,
       "cc": 51.95379963898918,
       "cc_by_type": {
        "score": 51.95379963898918
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 51.95379963898918
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.332352941176471,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": 1.0029411764705842
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.7083333333333334,
       "cc": 51.67661411052716,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100.0,
        "score": -7.703076923076924
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 51.67661411052716
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 44.13327913279132,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50.0,
        "score": -1.3400000000000079
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.13327913279132
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -19.73008040935672,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -20.59374999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 34.75663647342995,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 66.58874999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 34.75663647342995
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -33.22037735849057,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -182.68
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 58.02814264167205,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 50.835128205128186
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 58.02814264167205
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 26.271241226076857,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50.0,
        "score": 60.17260869565217
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 26.271241226076857
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 4.0666666666666655,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60.0,
        "score": 55.533333333333324
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 4.0666666666666655
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 50.038830921636055,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 33.33333333333333,
        "score": 42.4790909090909
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 50.038830921636055
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 50.424990603711535,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 29.569545454545477
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 50.424990603711535
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 10.750358422939065,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 27.949999999999996
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 10.750358422939065
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 64.90648148148148,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 74.71944444444443
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 64.90648148148148
      },
      "model_routing": {
       "accuracy": 0.8894009216589862,
       "cc": 77.00875755321067,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 87.87878787878789,
        "score": 48.99164062499999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 77.00875755321067
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 19.810444093184298,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 70.94764705882352
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 19.810444093184298
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 57.81635752208175,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.5419230769231
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.81635752208175
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 35.491342606007926,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 21.25235294117648
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 35.491342606007926
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 19.471690821256033,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 35.22666666666665
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 19.471690821256033
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 97.71456140350877,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 93.14368421052632
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.71456140350877
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.10373154216826,
      89.78560246866702
     ],
     "I": [
      50.35029169471709,
      59.53366527085479
     ],
     "capability": [
      68.38797728474432,
      73.87563540528951
     ],
     "composite": {
      "A": [
       5.197359394601136,
       5.376101341058595
      ],
      "B": [
       5.41156944489018,
       5.712684941761912
      ],
      "C": [
       3.656661809888081,
       5.28767825747465
      ]
     },
     "seed": 52
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.11821147102742,
     "capability": 72.28855576872621,
     "intelligence": 55.45890006642501
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 91.00629508196715,
      "cc": 75.37266308195717,
      "cc_by_type": {
       "choice": 75.37266308195717
      },
      "cc_tiered": 76.20412948610885,
      "cc_tiered_open": 78.56754551019564,
      "cc_tiered_sealed": 73.84071346202205,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.37266308195717
     },
     "noul": {
      "accuracy": 0.6746666666666666,
      "calibration": 84.24203753351209,
      "cc": 34.93333333333333,
      "cc_by_type": {
       "noul": 34.93333333333333
      },
      "cc_tiered": 34.898180567372094,
      "cc_tiered_open": 42.275362318840585,
      "cc_tiered_sealed": 27.520998815903607,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 34.93333333333333
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 92.10630179760304,
      "cc": 51.95379963898918,
      "cc_by_type": {
       "score": 51.95379963898918
      },
      "cc_tiered": 55.27439014579409,
      "cc_tiered_open": 62.26110671688638,
      "cc_tiered_sealed": 48.2876735747018,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.95379963898918
     }
    },
    "raw_on_A": {
     "calibration": 85.46331356109388,
     "capability": 70.8690106210405,
     "intelligence": 56.274707680987134
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 4.508299157053411,
    "Emphasis on Accuracy 60:20:20": 5.285169908639133,
    "Emphasis on Cost 20:20:60": 3.260249454005761,
    "Emphasis on Speed 20:60:20": 5.899504756880868,
    "Intelligence only": 7.12748331584751
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "baselabs (Base Labs at Baseten; baselabs on Hugging Face)",
   "axes": {
    "calibration": 94.26152926942419,
    "cost": 39.535093444670714,
    "intelligence": 76.20447479084162,
    "speed": 84.40895777972253
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.024914839344262127,
      "ece_score": 95.01703213114757,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 95.01703213114757,
      "score_before_time_limit": 95.01703213114757,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.10723860589812328,
      "brier": 0.04332660199162734,
      "ece": 0.027278206434316478,
      "n": 373,
      "score": 94.5443587131367,
      "score_before_time_limit": 94.5443587131367,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9865591397849462,
      "interval90_width": 0.2329749103942652,
      "n": 372,
      "nrps": 0.04045206609657794,
      "score": 93.22319696398831,
      "score_before_time_limit": 93.22319696398831,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.04754199731182791
     }
    },
    "score": 94.26152926942419
   },
   "capability": 85.2330020301329,
   "composite_ci95": {
    "A": [
     40.357068538808186,
     41.357775060168485
    ],
    "B": [
     41.25507367988368,
     42.80143010392023
    ],
    "C": [
     40.357068538808186,
     41.357775060168485
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the rates for this base string are identical in the earlier module 81e3578f): base string Qwen/Qwen3.5-27B (the card's base_model) priced as qwen/qwen3.5-27b USD 0.195/M input, USD 1.56/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of OneJev's server (create_app, behind our launcher; input_tokens = prefix plus all suffix tokens, incl. system prompt and chat template; output_tokens = number of questions = 1 per answered single-question item, by design, nothing generated). Our offline count through a first-party replica of the engine's pre-GPU path (bl_render.py) and the pinned tokenizer predicted the server's numbers EXACTLY, item by item: 777,323 input tokens and 1,485 output tokens over the 1,485 answered items, and 15 HTTP 422 refusals at the pre-registered ids and positions. OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.103633 per 1,000 decisions on the common basis (USD 0.102596 over all 1,500 items) is 1.604x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them); their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.10259639000000002
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.1036327171717172
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "baselabs mammoth-1 (27B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. The launcher and engine run in a docker container with network none (cut proven from inside before any item hash or model load: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only model and input, empty credential variables, HF offline flags, one request at a time, timed over loopback on the VM. Peak CUDA memory allocated after the scored run 59,375 MiB of 97,887 MiB. Measured 22 Oct 2026 (row 09:57-10:06Z, scored run 401 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked on the node and inside the container). This row's input was shredded from the node and its absence verified, and the VM was removed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.0375549583495,
    "I_sealed": 74.37139462333373,
    "base": 76.20447479084162,
    "excess": -6.173366435339936,
    "gap": 3.6661603350157748,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 88.89680314176312,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 88.67568417741427,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 72.17046238785369,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 73.04539934543172,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 77.82599696969696,
       "hard": 80.89890462962963,
       "judge": 74.66825932203389,
       "standard": 52.51379999999999
      }
     },
     "sealed|choice": {
      "cc": 82.91172191752877,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 77.62229675031803,
       "judge": 82.37022526934379,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 72.33456301804218,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 67.27272727272727,
       "judge": 75.38461538461539,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 67.86789893443024,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 85.15065087719297,
       "hard": 64.31989746478874,
       "judge": 68.15543802083333,
       "standard": 65.89121727272727
      }
     }
    }
   },
   "jevbench_score": 41.08325309769789,
   "key": "mammoth-1",
   "last_measured_on": "2026-10-22",
   "licence": "MIT (model card metadata since 21 Oct 2026; no LICENSE file; base Qwen/Qwen3.5-27B is Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "baselabs/mammoth-1@03007ab3c4481bf01c71ef63adbbbc4b2093d71a",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.972972972972973,
    "decisive": 333,
    "decisive_rate": 0.8927613941018767,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 90,
   "ranked": true,
   "ranks": {
    "A": 90,
    "B": 82,
    "C": 73,
    "capability": 7
   },
   "repo": "https://huggingface.co/baselabs/mammoth-1",
   "scores": {
    "A": 41.08325309769789,
    "B": 42.24672049685249,
    "C": 41.08325309769789
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5256244310000057,
    "p50_s_raw": 0.18781221550000282,
    "p95_s_adjusted": 0.6893323799000428,
    "p95_s_raw": 0.2696661899500214
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The authors' own serving runtime: OneJev qev package at 81ce62f1597c91e46767d4d02d6ac2e18534fe94 plus the authors' serve.patch (sha256 f234c125f1d38d82104b9e1f24e75bde51e4401928acfba2fbf5c59ef2fb46b8; 15 qev/ files, manifest sha256 818ca0fc1d26ec6a2369aac210fb1de41d908a9e52b45e25bc6150714f046501), vendored as byte-pinned files (re-verified inside the smoke container: 15 files OK), no git clone, no pip -e, setup.sh never run. It is run through OUR first-party launcher baselabs_launcher.py (sha256 7a0896373a6677034909ffa56f17558cbd9ceaf389a9dafdea59cfc638ce26ed), which builds the authors' qev.engine.DecisionEngine (bfloat16 + fp32 LM head, block-bidirectional attention on the full-attention layers, slot_lm readout, 256 slots, debias 1, no calibration file so T=1, auto fork mode which ran as 'batched', author limits 32,768 / 65,536 tokens) and hands it to OneJev's own create_app on 127.0.0.1:8000. `qev serve` was NOT used: it routes these checkpoints (config has a vision_config) to MMDecisionEngine, which never attaches the block-bidirectional mask; the DecisionEngine is what the authors' own Decision Index runs use. The launcher adds only the authors' own 32BitIndexMath sequential-fork fallback and one read-only /_launcher/info route; no prompt, truncation, calibration, answer or usage change. After the scored run the engine reported noncausal attention with the mask installed and every counted mask build (3,000 of 3,000) carried block ids; the counters show that the mask was built with block ids, not numerical parity with the Decision Index runs or with a causal-mask comparison (neither was run). Code gate: cleared by the benchmark owner (22 Oct 2026 08:42Z); a later addendum covering the attention-mask monkeypatch, the Hub-kernel path and the setup.sh extras was disclosed to the owner afterwards and no fresh confirmation was claimed. DISCLOSED CODE PATTERNS (download/client/kernel paths are inactive; the attention-mask monkeypatch executes offline as described below): setup.sh (git clone, pip install -e, a CUDA extension build that can download a wheel), qev/client.py (urllib HTTP client class, unused), qev/engine.py and mm_engine.py Hub loaders (a local verified snapshot is passed, offline flags set), readout.py torch.load of a head file (checkpoints say slot_lm, no head file), constraints.txt listing wandb and an OpenTelemetry exporter stack (not installed), an in-process monkeypatch of transformers' create_causal_mask to add block ids (readout.py:104-124, runs inside the network-none container), and transformers' optional Hub-kernel path (inactive: the kernels package is not installed). A static source review and two independent runner reviews preceded the run (their one must was closed and re-checked before launch). Image: our own build on the pod (network on, before any sealed byte) from pytorch/pytorch:2.8.0-cuda12.8-cudnn9-devel@sha256:a7103283ea7113e10ae5d014bd2342acebda0bc53164b2f7b1dd6eb7a766bdb6 plus 50 hash-pinned wheels (--require-hashes, lock sha256 bf9fa61b272ce83fe5a2ac4c6db219484f929083d46efe0f0a5252ab969c31da, installed with --no-index --no-deps); torch 2.8.0+cu128, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2; causal-conv1d is not installed (transformers uses its torch fallback); row image sha256:598f1f31ed5aa27f455caefcc3a509e2140bcaf49cae685db419f8fecd5f4682. Weights: baselabs/mammoth-1 at 03007ab3c4481bf01c71ef63adbbbc4b2093d71a, the 10 serving files (54,733,723,313 bytes, serve manifest sha256 33025dc9...), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file on the host and again in the smoke container; README, engine.py, setup.sh and the other author files were not used. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry; the GitHub request for this model came from a third party, not the authors). AUTHOR-DEFAULT CONTEXT LIMITS: the engine allows 32,768 tokens for state plus the longest question (strict >) and 65,536 per request and raises an error above that, which the authors' runtime returns through OneJev create_app as HTTP 422, never a truncation; the authors train at 16,384 tokens. The 15 items of 76,374-80,534 tokens were refused exactly as pre-registered and are scored wrong; the largest answered item is 1,854 tokens, inside the trained length. NOT AN AUTHOR-SHIPPED SERVER: the authors ship no server file; our first-party launcher around their DecisionEngine and OneJev's web app is what was timed (see code_revision). SERIAL ONLY: the server handles one request at a time and our driver sent one at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). causal-conv1d is not installed, so transformers' slower torch convolution fallback ran. LICENCE: MIT in the card metadata since 21 Oct 2026 (commit 9d1962f1 changed 'proprietary' to 'mit' and removed the evaluation-only line); there is no LICENSE file, no copyright holder and no carried Qwen attribution, and the Decision Index row still said 'proprietary' on 22 Oct (stale). Weights are identical to the Decision Index-scored revision 9d1962f1 (the earlier 68.87 row was another weight set and is not used). TRAINING DATA: not disclosed (the Decision Index notes it); the Decision Index states that its checks found signs that the model's training data overlapped one private new-domain test, which it does not count for this model, we cannot rule out overlap on other tests (our inference); the authors' technical report says decontamination against the Jev / Artificial Analysis index benchmarks was done (self-reported, not verified here); we ran no overlap check and no JevBench use or denial is declared. Our own public-versus-sealed check on this row: Intelligence 78.04 on the public items against 74.37 on the sealed ones, a gap of 3.67 points, so no penalty applies. SELF-REPORTED, unverified: Decision Index 0.4 score 71.24 and an Artificial Analysis replica score of 65.9 on the card; the DI lab's own Decision Index 0.4.1 full score is 68.44 (public 75.34, decision agreement with the submitted rows 99.85% on 831 rows against a 98.5% bar), DI-measured latency median 220 ms. 10 s RULE: max measured decision latency 5.78 s (one outlier; all other decisions <=0.49 s; median 0.19 s, p95 0.27 s); 0 decisions over 10 s; a 20k-token request took about 5.7 s warm and a 32k-token request about 17 s in the smoke, but no answered item is that large.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 95.74752222222223,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 95.5759
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 95.74752222222223
      },
      "calibration_gen": {
       "accuracy": 0.801980198019802,
       "cc": 68.72396086183281,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 68.0,
        "score": 63.622571875
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 68.72396086183281
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 89.70989044194289,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 100.0,
        "score": 72.38170384615384
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.70989044194289
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.14344259259259,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 98.43032777777778
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.14344259259259
      },
      "jaggedness": {
       "accuracy": 0.75,
       "cc": 58.27060204744177,
       "cc_by_type": {
        "choice": 71.12608277189607,
        "noul": 35.13513513513513,
        "score": 68.55058823529411
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.27060204744177
      },
      "llm_router": {
       "accuracy": 0.9361022364217252,
       "cc": 84.22002665949906,
       "cc_by_type": {
        "choice": 93.66029761380645,
        "noul": 93.81443298969072,
        "score": 65.185349375
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.22002665949906
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 67.83860883652432,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75.0,
        "score": 65.62922857142858
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 67.83860883652432
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 70.50534358550196,
       "cc_by_type": {
        "choice": 85.32768177372026,
        "noul": 55.55555555555556,
        "score": 70.63279342723004
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.50534358550196
      },
      "negation_pair_gen": {
       "accuracy": 0.9418604651162791,
       "cc": 85.3283288888889,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 86.66666666666667,
        "score": 69.31832000000001
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 85.3283288888889
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.953125,
       "cc": 89.05839415184414,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100.0,
        "score": 73.18118846153845
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 89.05839415184414
      },
      "usecase": {
       "accuracy": 0.8136363636363636,
       "cc": 73.00379157908498,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 66.66666666666667,
        "score": 74.4035316
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 73.00379157908498
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 75.93615006105006,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 46.856069230769236
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 75.93615006105006
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 84.03880666666666,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 52.116419999999984
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 84.03880666666666
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 89.71216172887007,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100.0,
        "score": 77.10108695652174
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 89.71216172887007
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 60.97645124146493,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 33.33333333333333,
        "score": 99.87535000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 60.97645124146493
      },
      "en": {
       "accuracy": 0.8618592528236316,
       "cc": 77.0320042250382,
       "cc_by_type": {
        "choice": 84.8427527123174,
        "noul": 75.53956834532374,
        "score": 70.71369161747343
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 77.0320042250382
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 76.33060235741931,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 38.77973529411766
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 76.33060235741931
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.55877777777776,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.34299999999998
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.55877777777776
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 85.39683890608876,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 98.54345789473685
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 85.39683890608876
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 78.93753282122906,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75.0,
        "score": 78.57237500000002
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 78.93753282122906
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 64.9473619047619,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": -5.15791428571426
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 64.9473619047619
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 61.87464,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 25.623919999999988
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 61.87464
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 16.17297142857142,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 48.51891428571427
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 16.17297142857142
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.76368333333333,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.29105
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.76368333333333
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 30.77286023088024,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 51.62594000000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 30.77286023088024
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 64.73543062130324,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 69.44116279069767
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 64.73543062130324
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 83.32287404591277,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100.0,
        "score": 71.18252903225806
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 83.32287404591277
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 43.79053510638298,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 47.96735000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 43.79053510638298
      },
      "tr": {
       "accuracy": 1.0,
       "cc": 94.68632916666667,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 84.0589875
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 94.68632916666667
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.58657849462365,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 95.0178
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.58657849462365
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 53.75864444444445,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 27.942600000000017
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 53.75864444444445
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8813333333333333,
       "cc": 84.67249660345587,
       "cc_by_type": {
        "choice": 84.67249660345587
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.67249660345587
      },
      "noul": {
       "accuracy": 0.864,
       "cc": 72.8,
       "cc_by_type": {
        "noul": 72.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 72.8
      },
      "score": {
       "accuracy": 0.792,
       "cc": 69.72138014440435,
       "cc_by_type": {
        "score": 69.72138014440435
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 69.72138014440435
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 49.03737058823529,
       "cc_by_type": {
        "choice": 84.0,
        "noul": 0.0,
        "score": 63.11211176470587
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 49.03737058823529
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 80.85620833731487,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 61.202165384615405
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.85620833731487
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 32.81459159891599,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100.0,
        "score": -69.0359
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 32.81459159891599
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 12.519774000974659,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": 29.372187500000013
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 12.519774000974659
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.614744112318846,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 74.49640625
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.614744112318846
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 30.680094654088066,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -49.46914999999996
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 30.680094654088066
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 65.21579784529196,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 82.2020153846154
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 65.21579784529196
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 60.91103279806091,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50.0,
        "score": 53.639021739130435
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 60.91103279806091
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 57.01100740740741,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 19.999999999999996,
        "score": 71.86635555555556
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 57.01100740740741
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 87.62383869314125,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100.0,
        "score": 75.71948181818182
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 87.62383869314125
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 80.56180541852633,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 75.53554545454546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 80.56180541852633
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 36.799901433691765,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 41.582500000000024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 36.799901433691765
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 62.91369907407407,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 68.74109722222221
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 62.91369907407407
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 79.61527939664502,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 90.90909090909092,
        "score": 53.78090312499999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.61527939664502
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 24.215866617741153,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 95.6802294117647
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 24.215866617741153
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 60.37083188105612,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 81.20534615384616
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 60.37083188105612
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 64.23455776103545,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50.0,
        "score": 72.26032352941176
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 64.23455776103545
      },
      "scientific_discovery": {
       "accuracy": 0.782608695652174,
       "cc": 72.78247748792269,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100.0,
        "score": 61.82569333333332
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 72.78247748792269
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.86276578947367,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.58829736842104
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.86276578947367
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      89.92645950017267,
      94.05049640628478
     ],
     "I": [
      72.3587407266138,
      79.89757564645463
     ],
     "capability": [
      81.55475983073057,
      86.27316534238307
     ],
     "composite": {
      "A": [
       40.357068538808186,
       41.357775060168485
      ],
      "B": [
       41.25507367988368,
       42.80143010392023
      ],
      "C": [
       40.357068538808186,
       41.357775060168485
      ]
     },
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 94.26152926942419,
     "capability": 85.2330020301329,
     "intelligence": 76.20447479084162
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8813333333333333,
      "calibration": 95.01703213114757,
      "cc": 84.67249660345587,
      "cc_by_type": {
       "choice": 84.67249660345587
      },
      "cc_tiered": 85.90426252964593,
      "cc_tiered_open": 88.89680314176312,
      "cc_tiered_sealed": 82.91172191752877,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.67249660345587
     },
     "noul": {
      "accuracy": 0.864,
      "calibration": 94.5443587131367,
      "cc": 72.8,
      "cc_by_type": {
       "noul": 72.8
      },
      "cc_tiered": 72.25251270294794,
      "cc_tiered_open": 72.17046238785369,
      "cc_tiered_sealed": 72.33456301804218,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 72.8
     },
     "score": {
      "accuracy": 0.792,
      "calibration": 93.22319696398831,
      "cc": 69.72138014440435,
      "cc_by_type": {
       "score": 69.72138014440435
      },
      "cc_tiered": 70.45664913993099,
      "cc_tiered_open": 73.04539934543172,
      "cc_tiered_sealed": 67.86789893443024,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 69.72138014440435
     }
    },
    "raw_on_A": {
     "calibration": 91.72815618560855,
     "capability": 83.14098848301494,
     "intelligence": 74.55382078042132
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 37.315795851806335,
    "Emphasis on Accuracy 60:20:20": 40.85864409923797,
    "Emphasis on Cost 20:20:60": 30.996498893569974,
    "Emphasis on Speed 20:60:20": 42.26797192977396,
    "Intelligence only": 47.64375742643189
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 91.36294095928801,
    "cost": 30.10714287562989,
    "intelligence": 70.95116002556868,
    "speed": 84.78525949557437
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05149823990803967,
      "ece_score": 89.70035201839207,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.58075154903422,
      "score_before_time_limit": 89.70035201839207,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.1554959785522788,
      "brier": 0.059426603690029486,
      "ece": 0.037544038564276544,
      "n": 373,
      "score": 92.49119228714468,
      "score_before_time_limit": 92.49119228714468,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9838709677419355,
      "interval90_width": 0.21415770609318993,
      "n": 372,
      "nrps": 0.04303092545751453,
      "score": 92.01687904168512,
      "score_before_time_limit": 92.01687904168512,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.058315746854391684
     }
    },
    "score": 91.36294095928801
   },
   "capability": 81.15705049242834,
   "composite_ci95": {
    "A": [
     20.34235168463289,
     20.906276175479466
    ],
    "B": [
     20.994022824015403,
     21.903444050989858
    ],
    "C": [
     20.34235168463289,
     20.906276175479466
    ]
   },
   "cost": {
    "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff for these weights): the frozen Qwen/Qwen3.8-27B market reference of the published v1.6 field through the frozen pricing_v15 module (reference() -> qwen/qwen3.8-27b, USD 0.42 per 1M input and USD 3.00 per 1M output; the same pair the published vllm-sr-d3, pplx-decider-v1-1-27b and torchcast-decision-27b rows carry) x this row's own measured tokens on the common 1,485-item basis (755,508 input tokens, counted by the author's own engine and equal per item to the offline count made before the timed run; the author's engine reports 0 output tokens because nothing is generated). USD 0.213679 per 1,000 decisions on the common basis (USD 0.2115422 over all 1,500 items), 3.308x the official cost cap USD 0.064595, so this row is outside the default cost-capped Capability ranking and the squared cost gate keeps its composite small. Estimated, not charged; not a quote from the author or any host (the rented GPU cost the evaluator USD 1.7726 for the whole run, which is not the row's price). COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which this row refuses too (HTTP 422, over its 32,768-token serving limit); their tokens count for no row, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.213679 per 1,000 decisions (3.308x the official cost cap USD 0.064595), against USD 0.211542 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.21154224
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.2136790303030303
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nimble-v4-max (open weights, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM (instance type 1A100.22V, location FIN-03, Finland): 1x NVIDIA A100-SXM4-80GB 80 GB, driver 580.178.04, torch 2.8.0+cu128. A different card from the 1x H100 80 GB on which most of this draw's other GPU rows were timed, so this row's Speed axis rests on different hardware (A100 is slower than H100). The author's own production engine nimble.evaluation.decision_index_engine:NimbleEngine (bespokelabs/nimble-v4-max-decision-index @ adea14289afa29dca3ff38b1fcb76e58f6add230, code/nimble/, unmodified) run in process behind a first-party HTTP shim (evaluator code; it only relabels a score answer's type, adds output_tokens 0 and turns the engine's Unsupported into HTTP 422), driven by the frozen run_v15 harness and the unchanged TypeSafe adapter. Engine options are the author's own environment.json settings, not tuned: shared_prefix on, max_tails 48, max_tokens 32,768. Weights bespokelabs/Nimble-v4-max @ 655aa7051c2ab39660193ebb70c8926d3d2cb7aa (11 safetensors shards hash-verified on the pod), base and tokenizer Qwen/Qwen3.8-27B @ 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0. Isolation: fresh container with --network none (the cut proven from inside before the input was read), read-only root filesystem, all capabilities dropped, no credentials, Hub offline mode; 1,500 items in one pass, no resume, one sealed use. 15 items were refused with HTTP 422 because their prompts (76,358-80,516 input tokens) exceed the 32,768-token serving limit; nothing was truncated, and the refused set equals the set predicted by the offline count before the run. They count as wrong in every axis (full 1,500 denominator). The standard self-hosted x2 + 0.15 s own-server adjustment (an assumption, not measured) is applied on top of the raw p50/p95. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b); VM deleted afterwards (deletion verified).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.69513917272718,
    "I_sealed": 68.20718087841018,
    "base": 70.95116002556868,
    "excess": -4.3515684760387146,
    "gap": 5.487958294316996,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 81.35026138087967,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 88.13447593935399,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 69.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 69.7869159716704,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.04039975193228,
       "hard": 75.99272722221137,
       "judge": 76.32465627204957,
       "standard": 48.44194112988871
      }
     },
     "sealed|choice": {
      "cc": 80.44964453054286,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 72.41817971550824,
       "judge": 82.37022526934379,
       "standard": 85.76249736342545
      }
     },
     "sealed|noul": {
      "cc": 57.42375894820568,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 47.272727272727266,
       "judge": 60.00000000000001,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 66.74813915648204,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.93672034384494,
       "hard": 63.11277519836356,
       "judge": 66.14788845209046,
       "standard": 65.82495253562494
      }
     }
    }
   },
   "jevbench_score": 20.704204380128797,
   "key": "nimble-v4-max-open",
   "last_measured_on": "2026-10-21",
   "licence": "CC BY-NC 4.0 (weights, non-commercial); base Qwen/Qwen3.8-27B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "bespokelabs/Nimble-v4-max@655aa7051c2ab39660193ebb70c8926d3d2cb7aa",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9650793650793651,
    "decisive": 315,
    "decisive_rate": 0.8445040214477212,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 141,
   "ranked": true,
   "ranks": {
    "A": 141,
    "B": 135,
    "C": 123,
    "capability": 26
   },
   "repo": "https://huggingface.co/bespokelabs/Nimble-v4-max",
   "scores": {
    "A": 20.704204380128797,
    "B": 21.545234515512576,
    "C": 20.704204380128797
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4579101519997494,
    "p50_s_raw": 0.15395507599987468,
    "p95_s_adjusted": 0.725594159600314,
    "p95_s_raw": 0.287797079800157
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "bespokelabs/nimble-v4-max-decision-index @ adea14289afa29dca3ff38b1fcb76e58f6add230 (code/nimble/, unmodified); evaluator HTTP shim. MANDATORY DISCLOSURE: measured on a draw the author's hosted endpoint received on 14 Oct; weights committed 18 Oct. Second and last use by this author on this draw. This row is not an independent corroboration of the hosted row nimble-v4-max: the two rows come from the same author, on different hardware through a different prompt path, and the hosted row stays a separate row. We cannot verify that these weights are the bytes served by the hosted endpoint on 14 Oct, or that the exposed items played no role in producing them. Licence CC BY-NC 4.0 (weights, non-commercial). The 15 long items are refused (HTTP 422 from the engine's serving limit) and count as wrong in every axis (full 1,500 denominator). Output tokens are 0 on every answered item (the engine has no generate loop); kept as sent.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 77.50539758799437,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 40.8495260973164
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 77.50539758799437
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 55.722131664662925,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 43.99999999999999,
        "score": 58.16109276705346
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 55.722131664662925
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 87.08324700297233,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 75.16121345697486
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 87.08324700297233
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.23953564456099,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 98.71860693368296
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.23953564456099
      },
      "jaggedness": {
       "accuracy": 0.7611111111111111,
       "cc": 60.810812329847664,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 35.13513513513513,
        "score": 80.5023066667274
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 60.810812329847664
      },
      "llm_router": {
       "accuracy": 0.939297124600639,
       "cc": 86.01641161305686,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 100.0,
        "score": 62.80401162881576
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 86.01641161305686
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 39.15407158120296,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 0.0,
        "score": 54.57561680546456
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 39.15407158120296
      },
      "multilingual": {
       "accuracy": 0.7454545454545455,
       "cc": 59.064742600566326,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": 38.888888888888886,
        "score": 64.71551172011365
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 59.064742600566326
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 79.53360362848692,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 66.66666666666667,
        "score": 71.93414421879409
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 79.53360362848692
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 77.78784655702943,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 80.0,
        "score": 68.3785546861033
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 77.78784655702943
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 62.28583292370893,
       "cc_by_type": {
        "choice": 75.49019607843138,
        "noul": 41.66666666666667,
        "score": 69.70063602602873
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 62.28583292370893
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 69.44413742315277,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 27.380031317077336
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 69.44413742315277
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.5384615384615384,
       "cc": 33.095116601756565,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 7.4266708498165634
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 33.095116601756565
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 83.20266789460061,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100.0,
        "score": 81.46641076344784
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.20266789460061
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.49330391035204,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 99.99946480368183
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.49330391035204
      },
      "en": {
       "accuracy": 0.841876629018245,
       "cc": 73.72062158622043,
       "cc_by_type": {
        "choice": 82.64605020685615,
        "noul": 67.62589928057554,
        "score": 70.88991527122961
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 73.72062158622043
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 69.1563675244238,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 17.25703079513109
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.1563675244238
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.6143991192104,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.50986402429791
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.6143991192104
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 89.90152600011213,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 90.88104858857166
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 89.90152600011213
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 51.65101054842187,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25.0,
        "score": 46.71280818157846
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 51.65101054842187
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 72.48648788016195,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 31.489081723104707
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 72.48648788016195
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 31.013928391091792,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0.0,
        "score": 21.6359074925764
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 31.013928391091792
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 5.767945316873512,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 61.748280395064995
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 5.767945316873512
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.74044306637369,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 99.88799586578774
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.74044306637369
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -35.64365064350917,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100.0,
        "score": -54.982899982475544
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 78.61170275794112,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80.0,
        "score": 71.06997920061133
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 78.61170275794112
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 65.31808698901088,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 67.77512130881225
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.31808698901088
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 56.56363949454061,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 60.75474827085587
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 56.56363949454061
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 74.1488039926556,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 44.185542412749435
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 74.1488039926556
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 74.07269722878466,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 86.73422071861204
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 74.07269722878466
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 52.22341428333251,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 23.336909516664207
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 52.22341428333251
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8533333333333334,
       "cc": 81.0558946784286,
       "cc_by_type": {
        "choice": 81.0558946784286
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.0558946784286
      },
      "noul": {
       "accuracy": 0.8106666666666666,
       "cc": 62.133333333333326,
       "cc_by_type": {
        "noul": 62.133333333333326
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.133333333333326
      },
      "score": {
       "accuracy": 0.768,
       "cc": 68.17088476320109,
       "cc_by_type": {
        "score": 68.17088476320109
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 68.17088476320109
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 23.9901133999504,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -50.0,
        "score": 37.970340199851194
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 23.9901133999504
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 86.05864331437924,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 76.80947031580855
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 86.05864331437924
      },
      "customer_support": {
       "accuracy": 0.42857142857142855,
       "cc": 11.087714915362518,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 0.0,
        "score": 14.563957754217638
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 11.087714915362518
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -5.432351070271447,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": -67.7590415149079
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 42.193410468590194,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 88.8990719854807
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 42.193410468590194
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -18.85230943189946,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -164.7330289246292
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 43.47375956499048,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -14.28571428571429,
        "score": 54.510914549313185
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 43.47375956499048
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 67.45799134901468,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50.0,
        "score": 62.8269357195179
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 67.45799134901468
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 31.16919546982632,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60.0,
        "score": 74.34091974281229
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 31.16919546982632
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 71.82448200917604,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 74.50271083837747
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 71.82448200917604
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 66.73209462534784,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 56.268635297232194
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.73209462534784
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 24.466572747398118,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 36.84057845724812
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 24.466572747398118
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 70.42177274869952,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 62.781633025369196
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 70.42177274869952
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 81.8148451386316,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100.0,
        "score": 51.28869126005065
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.8148451386316
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 15.688592049548712,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100.0,
        "score": 81.61472048645805
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 15.688592049548712
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 59.91651322465041,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 79.84239018462905
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 59.91651322465041
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 36.12263173958099,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0.0,
        "score": 23.146220341895674
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 36.12263173958099
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 38.34717338014102,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0.0,
        "score": 73.0125346331767
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 38.34717338014102
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.94621362687455,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 99.83864088062363
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.94621362687455
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.36189364963293,
      92.35804441067566
     ],
     "I": [
      66.42332925082326,
      74.91599833906216
     ],
     "capability": [
      77.58373946084384,
      82.87729940235091
     ],
     "composite": {
      "A": [
       20.34235168463289,
       20.906276175479466
      ],
      "B": [
       20.994022824015403,
       21.903444050989858
      ],
      "C": [
       20.34235168463289,
       20.906276175479466
      ]
     },
     "seed": 56
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.36294095928801,
     "capability": 81.15705049242834,
     "intelligence": 70.95116002556868
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8533333333333334,
      "calibration": 89.58075154903422,
      "cc": 81.0558946784286,
      "cc_by_type": {
       "choice": 81.0558946784286
      },
      "cc_tiered": 80.89995295571126,
      "cc_tiered_open": 81.35026138087967,
      "cc_tiered_sealed": 80.44964453054286,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.0558946784286
     },
     "noul": {
      "accuracy": 0.8106666666666666,
      "calibration": 92.49119228714468,
      "cc": 62.133333333333326,
      "cc_by_type": {
       "noul": 62.133333333333326
      },
      "cc_tiered": 63.685999556918574,
      "cc_tiered_open": 69.94824016563147,
      "cc_tiered_sealed": 57.42375894820568,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.133333333333326
     },
     "score": {
      "accuracy": 0.768,
      "calibration": 92.01687904168512,
      "cc": 68.17088476320109,
      "cc_by_type": {
       "score": 68.17088476320109
      },
      "cc_tiered": 68.26752756407622,
      "cc_tiered_open": 69.7869159716704,
      "cc_tiered_sealed": 66.74813915648204,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.17088476320109
     }
    },
    "raw_on_A": {
     "calibration": 89.46067847128532,
     "capability": 79.60532503979249,
     "intelligence": 69.74997160829966
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.40381083172275,
    "Emphasis on Accuracy 60:20:20": 20.768041285916023,
    "Emphasis on Cost 20:20:60": 14.44148161077433,
    "Emphasis on Speed 20:60:20": 21.92317130047886,
    "Intelligence only": 25.72518927700611
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ironbcc (Hugging Face)",
   "axes": {
    "calibration": 83.5320529378626,
    "cost": 48.88742799396291,
    "intelligence": 46.11278573865326,
    "speed": 91.9752040844015
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09739173773645653,
      "ece_score": 80.5216524527087,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.5216524527087,
      "score_before_time_limit": 80.5216524527087,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.30294906166219837,
      "brier": 0.13463719291769388,
      "ece": 0.0866352733474072,
      "n": 373,
      "score": 82.67294533051856,
      "score_before_time_limit": 82.67294533051856,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9354838709677419,
      "interval90_width": 0.3573028673835125,
      "n": 372,
      "nrps": 0.09039978855699243,
      "score": 87.40156103036057,
      "score_before_time_limit": 87.40156103036057,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.08078449541789799
     }
    },
    "score": 83.5320529378626
   },
   "capability": 64.82241933825793,
   "composite_ci95": {
    "A": [
     35.74446302889502,
     60.242978818625105
    ],
    "B": [
     32.723177939313004,
     57.30964841095409
    ],
    "C": [
     24.822543770065984,
     42.12415462908831
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base Qwen/Qwen3.5-9B (declared by the author on the card and in merge_record.json) -> qwen/qwen3.5-9b USD 0.10/M input, USD 0.15/M output. The same estimate as the v1 row seb-9b. Estimated, not charged, not a quote from the author or any host. Tokens as reported by vLLM's own usage block: 748,494 input tokens on the 1,485 answered items (equal to our offline per-item count item by item, 0 mismatches, pre-registered; the same total as the v1 row, as the tokenizer and template are unchanged), and 1 output token per answered row (1,485: one forward pass that reads the label logprobs); the 15 refused items count 0 tokens. COST ON THE COMMON BASIS: USD 0.050554 per 1,000 decisions on 1,485 items, 0.783 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0500481
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.050553636363636356
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Seb-9B-v2 (ironbcc, Qwen3.5-9B fine-tune, one forward pass, stock vLLM, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.574/h. Stock vLLM 0.30.0 (127.0.0.1:8000, digest-pinned image) and our shim (the /v1/systemone endpoint) run in containers with network none, driven one request at a time by the unchanged pinned pod_driver_v16. The standard self-hosted x2 + 0.15 s latency adjustment applies. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 07:12-07:25Z; measurement wrapper 250 s; scored loop 63.3 s per the receipt) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:25Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 53.38140037125771,
    "I_sealed": 38.844171106048805,
    "base": 46.11278573865326,
    "excess": 4.697702494853196,
    "gap": 14.537229265208907,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 71.25353822050559,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 64.1396665618119,
       "judge": 68.35860250494397,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 38.33471359558317,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 50.55594929768438,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.48140097601048,
       "hard": 47.61342904269908,
       "judge": 45.68452143173856,
       "standard": 55.28540576741062
      }
     },
     "sealed|choice": {
      "cc": 59.787341513336614,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 71.41041931385007,
       "hard": 53.16294668671215,
       "judge": 59.4515181194907,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 16.844494671566217,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 10.909090909090914,
       "judge": 4.615384615384621,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 39.90067713324359,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 69.80403626056912,
       "hard": 29.99445855403019,
       "judge": 36.563140254082136,
       "standard": 49.76774004674978
      }
     }
    }
   },
   "jevbench_score": 50.0499351781575,
   "key": "seb-9b-v2",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (LICENSE in the repo and HF metadata; base Qwen/Qwen3.5-9B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ironbcc/Seb-9B-v2@ca903824a11dd763a8e0dec5bce5e13e3e4166db",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9076923076923077,
    "decisive": 260,
    "decisive_rate": 0.6970509383378016,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 72,
   "ranked": true,
   "ranks": {
    "A": 72,
    "B": 75,
    "C": 81,
    "capability": 119
   },
   "repo": "https://huggingface.co/ironbcc/Seb-9B-v2",
   "scores": {
    "A": 50.0499351781575,
    "B": 46.90867423748439,
    "C": 34.756899429276054
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22097171999998863,
    "p50_s_raw": 0.03548585999999432,
    "p95_s_adjusted": 0.2871725307000929,
    "p95_s_raw": 0.06858626535004646
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "No author code runs at all (static review PASS: the repo's four serve/*.py files are excluded from the manifest and never copied to the node; the run fails if any .py/.sh/.so/.pyc/.pkl/.pt/.bin exists in /model). Weights: ironbcc/Seb-9B-v2 at ca903824a11dd763a8e0dec5bce5e13e3e4166db (Apache-2.0, public, not gated; Qwen/Qwen3.5-9B plus a merged LoRA, scale 20, 248 modules; 14 manifest files, 4 bf16 shards = 19,306,311,000 bytes; manifest sha256 d138599972ed6caa640987b3bd175a24e8f0d76e4485317d1de6240044d660d4; every file sha256-verified against the Hub oids). The Hub HEAD later moved to c01ef79eb54b491c83ed1260ab8444cbc662e65c by README, blog and media commits only: the 19 weight, config and tokenizer oids are identical at both commits, and the run loads exactly the pinned commit. config.json, chat_template.jinja, tokenizer.json, tokenizer_config.json and the preprocessor files carry the same git oids as the v1 row seb-9b; vocab.json and merges.txt on the Hub are truncated at 1,048,576 bytes (unused by the fast tokenizer; equivalence with v1 proven offline on 467 texts and by the usage match below). Served by the stock vLLM 0.30.0 OpenAI server (image pinned by digest vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90, no derived image) with the author's published command (vllm serve /model --served-model-name seb-9b-v2 --logprobs-mode processed_logprobs --max-logprobs 32 --generation-config vllm --max-model-len 32768 --dtype bfloat16 --max-num-seqs 128 --enable-prefix-caching --gpu-memory-utilization 0.85 --host 127.0.0.1 --port 8000); the only delta from the v2 card is --gpu-memory-utilization 0.85, kept from the v1 recipe (it only sizes the KV cache). vllm 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, tokenizers 0.23.2, flashinfer 0.6.18.post1. The /v1/systemone wire layer is OUR reviewed shim seb_systemone_r56.py (sha256 e617f911e9218359b35eb8d7761ae6674c75631398efdd76996b1741e6a17b99, byte-identical to the v1 row's): it builds the author's documented chat request (max_tokens 1, temperature 0, top_logprobs, structured_outputs.choice, thinking off) and renormalises exp(top_logprobs) over the option labels; it applies no temperature and no rule of its own. The runner was reviewed and findings resolved in the preregistration amendments before the run. FIRST MEASUREMENT OF THIS VERSION. NEW VERSION OF AN EXISTING MODEL: Seb-9B-v2 has new weights (all four shards differ from ironbcc/seb-9b); the v1 row seb-9b measured 20 Oct 2026 stays as it is. The submission reference is not recorded in the files read for this entry. Whether this row supersedes the v1 row or stands beside it is a release-owner decision. SECOND USE OF THE DRAW FOR THE FAMILY: evaluator exposure and device ruling admitted this as a same-author second use (v1 consumed the frozen draw on 20 Oct, on file; the v2 weights were published after that score), with disclosure; any further ironbcc row goes to the next draw. AUTHOR CONTEXT CAP: at the card's 32,768-token limit vLLM refuses the 15 items of this draw that are 76,344-80,504 tokens long (HTTP 400, mapped to a per-question refusal HTTP 422 by our shim) at positions 65, 169, 474, 482, 515, 573, 580, 729, 833, 899, 951, 1057, 1116, 1198, 1418, identical to the v1 set; they count as wrong. The card also says Seb-9B-v2 was trained and evaluated with prompts of up to 1,024 tokens and that --max-model-len only sets the server limit; answered items run up to 1,875 tokens (mean 504, median 455), so some answered items are longer than the author's stated training and evaluation length. We used the card's 32,768 setting, as for v1. OWN HTTP SHIM: the wire layer is ours (see code_revision). TRAINING DATA AND SELF-REPORTED NUMBERS (the author's own card, unverified): the card reports Decision Index 0.3 public 59.16 (against 57.96 for hosted Jev 1.13 and 60.70 for Jev 1.14); this is not a JevBench result. We did not audit training data or overlap with the JevBench public 231 and ran no overlap check; no JevBench-use statement for v2 was recorded in the files read. HARDWARE: unlike v1 (A100 80 GB), this row ran on one RTX PRO 6000 (sm_120); latency reflects that card and the Gated-DeltaNet kernels ran clean (usage matched our offline count on all 1,485 answered items). 10 s RULE: max measured decision latency 0.138 s, 0 decisions over 10 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, max probability difference 0. No outside request for this model. PREDECESSOR: seb-9b (/jev-models/seb-9b); both measurements remain separate rows.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 57.38641643007268,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 5.4925826235513675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 57.38641643007268
      },
      "calibration_gen": {
       "accuracy": 0.3564356435643564,
       "cc": 0.6457363133782437,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -19.999999999999996,
        "score": 1.4706129698271897
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.6457363133782437
      },
      "composite_gen": {
       "accuracy": 0.7623762376237624,
       "cc": 56.83985774960238,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 33.33333333333333,
        "score": 63.20250007807544
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 56.83985774960238
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 78.79499429934914,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 76.38498289804741
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 78.79499429934914
      },
      "jaggedness": {
       "accuracy": 0.4388888888888889,
       "cc": 11.285698275626302,
       "cc_by_type": {
        "choice": 30.702598652550538,
        "noul": -35.13513513513513,
        "score": 38.2896313094635
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 11.285698275626302
      },
      "llm_router": {
       "accuracy": 0.8274760383386581,
       "cc": 71.57773436993652,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 69.0721649484536,
        "score": 63.88768252166237
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.57773436993652
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -29.89981458777572,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100.0,
        "score": -15.472639639615803
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.68,
       "cc": 45.823066202819945,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 25.0,
        "score": 46.70460780311259
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 45.823066202819945
      },
      "negation_pair_gen": {
       "accuracy": 0.627906976744186,
       "cc": 30.330265934353008,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": 0.0,
        "score": 14.32413113639237
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 30.330265934353008
      },
      "option_order": {
       "accuracy": 0.7916666666666666,
       "cc": 73.36884154460719,
       "cc_by_type": {
        "choice": 73.36884154460719
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 73.36884154460719
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 83.12848382106596,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100.0,
        "score": 73.4094754872219
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.12848382106596
      },
      "usecase": {
       "accuracy": 0.6227272727272727,
       "cc": 40.88753807810426,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 16.666666666666675,
        "score": 45.211633842155884
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 40.88753807810426
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 4.49553438461395,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50.0,
        "score": -79.370539703301
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 4.49553438461395
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 55.80691686339742,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 54.824744445799034
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 55.80691686339742
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 68.11596222240989,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 53.2416919769642
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.11596222240989
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.079887607786528,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.94543563527765
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.079887607786528
      },
      "en": {
       "accuracy": 0.6533449174630755,
       "cc": 43.67778416055842,
       "cc_by_type": {
        "choice": 61.77737640497429,
        "noul": 25.179856115107913,
        "score": 44.076119961593044
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 43.67778416055842
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 64.52391114382674,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100.0,
        "score": 22.935518097059315
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 64.52391114382674
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 48.30465437401824,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 44.36751503462304
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 48.30465437401824
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 82.37184101564326,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 89.46846422340035
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 82.37184101564326
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 18.888583868421673,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -50.0,
        "score": 40.18530467789072
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 18.888583868421673
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 85.61439800351773,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 70.87281209317207
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 85.61439800351773
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 32.49832518137121,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 34.683220182715694
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 32.49832518137121
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 27.528579980368807,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0.0,
        "score": 27.03018438555087
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 27.528579980368807
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 93.79443805963497,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 98.04998084557158
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 93.79443805963497
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 18.84465359898142,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 56.53396079694426
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.84465359898142
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 30.861661394956045,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 27.81985511165611
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 30.861661394956045
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 73.76369289597659,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 63.71889247696926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.76369289597659
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 33.91312705331635,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 58.33512584080012
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 33.91312705331635
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 36.4780826755024,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": -68.8266215387102
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 36.4780826755024
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 71.97310415505284,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 80.43544149741659
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 71.97310415505284
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 21.49023182592421,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -68.8626378555607
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 21.49023182592421
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.712,
       "cc": 62.80066591400524,
       "cc_by_type": {
        "choice": 62.80066591400524
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 62.80066591400524
      },
      "noul": {
       "accuracy": 0.6293333333333333,
       "cc": 25.86666666666666,
       "cc_by_type": {
        "noul": 25.86666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 25.86666666666666
      },
      "score": {
       "accuracy": 0.584,
       "cc": 43.440386235078385,
       "cc_by_type": {
        "score": 43.440386235078385
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 43.440386235078385
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 22.47089665932023,
       "cc_by_type": {
        "choice": 52.0,
        "noul": 25.0,
        "score": -9.587310022039318
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 22.47089665932023
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 1.9727743196350314,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": -4.847722589749148
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.9727743196350314
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 68.48603888832874,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 87.97844186823825
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 68.48603888832874
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -7.921297584720989,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": -8.559214391589865
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 33.47949224798697,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 62.757317323671046
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 33.47949224798697
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -44.26660553570567,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 0.0,
        "score": -157.32811849390947
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 43.05378535028399,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 44.28740647102004
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 43.05378535028399
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": 4.9222426772100825,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50.0,
        "score": 37.93745973894732
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 4.9222426772100825
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 20.122783190160334,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 82.868349570481
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 20.122783190160334
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 33.38482637089782,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 0.0,
        "score": 38.69837632896756
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.38482637089782
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 48.04315272638223,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 11.111111111111116,
        "score": 33.01834706803558
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 48.04315272638223
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -21.665428432461855,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": -69.29736056620277
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 46.309405333434256,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 19.999999999999996,
        "score": 53.47716033811467
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 46.309405333434256
      },
      "model_routing": {
       "accuracy": 0.847926267281106,
       "cc": 73.4346855974761,
       "cc_by_type": {
        "choice": 89.48051948051949,
        "noul": 78.78787878787878,
        "score": 52.035658524030026
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 73.4346855974761
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 1.6124700570809363,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 50.90266928832534
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 1.6124700570809363
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 80.20609025286802,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100.0,
        "score": 57.331362680609644
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 80.20609025286802
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 27.437855304825547,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": 11.870216160782043
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 27.437855304825547
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 22.51872904218071,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 44.36778132944068
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 22.51872904218071
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 99.0851520625459,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.25545618763773
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.0851520625459
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.71560663717828,
      85.21517762668637
     ],
     "I": [
      39.989963245688045,
      50.10984922062006
     ],
     "capability": [
      59.63804732541866,
      66.85089580077984
     ],
     "composite": {
      "A": [
       35.74446302889502,
       60.242978818625105
      ],
      "B": [
       32.723177939313004,
       57.30964841095409
      ],
      "C": [
       24.822543770065984,
       42.12415462908831
      ]
     },
     "seed": 70
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.5320529378626,
     "capability": 64.82241933825793,
     "intelligence": 46.11278573865326
    },
    "per_type": {
     "choice": {
      "accuracy": 0.712,
      "calibration": 80.5216524527087,
      "cc": 62.80066591400524,
      "cc_by_type": {
       "choice": 62.80066591400524
      },
      "cc_tiered": 65.5204398669211,
      "cc_tiered_open": 71.25353822050559,
      "cc_tiered_sealed": 59.787341513336614,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 62.80066591400524
     },
     "noul": {
      "accuracy": 0.6293333333333333,
      "calibration": 82.67294533051856,
      "cc": 25.86666666666666,
      "cc_by_type": {
       "noul": 25.86666666666666
      },
      "cc_tiered": 27.589604133574692,
      "cc_tiered_open": 38.33471359558317,
      "cc_tiered_sealed": 16.844494671566217,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.86666666666666
     },
     "score": {
      "accuracy": 0.584,
      "calibration": 87.40156103036057,
      "cc": 43.440386235078385,
      "cc_by_type": {
       "score": 43.440386235078385
      },
      "cc_tiered": 45.228313215463984,
      "cc_tiered_open": 50.55594929768438,
      "cc_tiered_sealed": 39.90067713324359,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.440386235078385
     }
    },
    "raw_on_A": {
     "calibration": 81.84002903415777,
     "capability": 65.22349885956152,
     "intelligence": 48.60696868496526
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 46.014101060953124,
    "Emphasis on Accuracy 60:20:20": 42.180841750659,
    "Emphasis on Cost 20:20:60": 43.28633845355636,
    "Emphasis on Speed 20:60:20": 54.383343919057616,
    "Intelligence only": 37.495438704806716
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Singularity Compute",
   "axes": {
    "calibration": 90.84828412834014,
    "cost": 64.26424619973979,
    "intelligence": 78.8866847061141,
    "speed": 85.30638666270315
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05436905982497981,
      "ece_score": 89.12618803500403,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.413178530724,
      "score_before_time_limit": 89.12618803500403,
      "time_limit_failed_share": 0.008
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.058022045174915465,
      "ece": 0.04121594125547194,
      "n": 375,
      "score": 91.26744208624478,
      "score_before_time_limit": 91.7568117489056,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.10022222222222223,
      "n": 375,
      "nrps": 0.04473236689969343,
      "score": 92.86423176805167,
      "score_before_time_limit": 93.61313686295532,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.041502447920600115
     }
    },
    "score": 90.84828412834014
   },
   "capability": 84.86748441722712,
   "composite_ci95": {
    "A": [
     76.93989597679156,
     79.39169870780026
    ],
    "B": [
     76.64350103296658,
     79.9065768646065
    ],
    "C": [
     76.93989597679156,
     79.39169870780026
    ]
   },
   "cost": {
    "basis": "Launch list price: the launch list price stated on the vendor's public product page (singularitycompute.com/super-decider, fetched 22 Oct 2026: USD 0.03 per 1M input tokens, output tokens free); the ASI:Cloud catalogue itself was not reachable to us. The page lists one price for the product and no separate rate for thinking mode; we apply it to both rows. Counted as an operator launch list price by the frozen pricing module pricing_v15 (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the launch-list line and the rates for this product are identical in the earlier module 81e3578f), which carries a launch-list-price line for this product. Applied to the tokens reported by the provider's API; not a base-model estimate. Over all 1,500 items the frozen resolve() gives USD 0.039197 per 1,000 decisions; the official cost axis uses the common 1,485-item basis, where it is USD 0.015531 per 1,000 decisions. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 768,771 input / 322,546 output tokens. All-item usage 1,959,826 input / 328,348 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03919652
    },
    "kind": "listed",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01553072727272727
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 11,
    "n_over_limit_answered_ok": 11,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Super Decider",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid fast-lane evaluation, published at the provider's request (the order was private and was made public at the customer's request). Vendor-hosted, first-party Singularity Compute API (llm.a.singularitycompute.com, POST /v1/systemone, model `super-decider`), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `singularitycompute` (no proxy) before the first request. All 1,500 of 1,500 requests answered with HTTP 200 and the served model was `super-decider` on every row; native probabilities on all three question types; one serial pass, no retries. Run with the provider's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.80985381401513,
    "I_sealed": 78.96351559821305,
    "base": 78.8866847061141,
    "excess": -9.993188554553633,
    "gap": -0.1536617841979222,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 82.86017015722445,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 88.13447593935399,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 84.04347826086956,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 82.6086956521739,
       "judge": 70.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 69.5259130239514,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 85.38361656115687,
       "hard": 81.92486849848484,
       "judge": 59.862361248832244,
       "standard": 51.294477968960514
      }
     },
     "sealed|choice": {
      "cc": 81.34124143256045,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 75.02023823291314,
       "judge": 80.60724779627816,
       "standard": 87.66083104830204
      }
     },
     "sealed|noul": {
      "cc": 83.99136221472814,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 78.18181818181817,
       "judge": 84.61538461538463,
       "standard": 97.01492537313432
      }
     },
     "sealed|score": {
      "cc": 71.55794314735057,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 88.87375089665288,
       "hard": 62.94798902769401,
       "judge": 75.64215151608957,
       "standard": 73.99363495890405
      }
     }
    }
   },
   "jevbench_score": 78.48219079083019,
   "key": "singularity-super-decider",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (Hugging Face model card; weights public, not gated); the card states that the licence is the same as that of the base model google/diffusiongemma-26B-A4B-it; the hosted service itself is proprietary",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:llm.a.singularitycompute.com/v1/systemone#super-decider",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9276139410187667,
    "decisive": 373,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1.0,
   "rank": 1,
   "ranked": true,
   "ranks": {
    "A": 1,
    "B": 1,
    "C": 1,
    "capability": 10
   },
   "repo": "https://huggingface.co/Singularity-Compute/super-decider",
   "scores": {
    "A": 78.48219079083019,
    "B": 78.56275738441518,
    "C": 78.48219079083019
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24612953246105462,
    "p50_s_raw": 0.24612953246105462,
    "p95_s_adjusted": 1.1972851804515812,
    "p95_s_raw": 1.1972851804515812
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no vendor code ran locally. Model, as pinned by the provider: Singularity-Compute/super-decider at 7f8b3f76e88aba6aa1159da75eb577ea990d2af1 on Hugging Face (public, not gated), a discrete-diffusion decision model fine-tuned from google/diffusiongemma-26B-A4B-it and served from an NVFP4 checkpoint (card statement). Serving code, as pinned by the provider: github.com/Singularity-Compute/super-decider at 10bf7eddcbd3b5f6dddf27e868231b2e1140dfba (pinned vLLM nightly with the provider's patches and a structured server). Neither the deployed weights nor the serving stack were independently inspected. MANDATORY DISCLOSURE: measured through the provider's hosted API on 21 Oct 2026; published at the provider's request. THINKING MODE: the API default (no `thinking` field, i.e. \"auto\"); the row shows one fixed setting, and the provider's docs say that in \"auto\" mode thinking is skipped when the server is busy, so the default row reflects the server load at the time of measurement (21 Oct 2026, 08:05-08:45 UTC). TRAINING AND SELECTION, provider statement: the public JevBench v1.4/v1.5 items were used as a validation set (recipe and checkpoint choice, serving parameters), not as training data, and the model card says no benchmark test items were used for training; the public set is therefore treated as seen during selection and the normal public-versus-sealed gap rule applies to this row. Provider statements are not independently verified.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1.0,
       "cc": 89.85545147257876,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 69.56635441773629
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 89.85545147257876
      },
      "calibration_gen": {
       "accuracy": 0.9702970297029703,
       "cc": 94.85874669366245,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92.0,
        "score": 95.75757624217506
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 94.85874669366245
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 85.67625328152366,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 77.77777777777777,
        "score": 95.5111446684192
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.67625328152366
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 99.12234530239441,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.36703590718322
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.12234530239441
      },
      "jaggedness": {
       "accuracy": 0.8722222222222222,
       "cc": 82.77262796695852,
       "cc_by_type": {
        "choice": 79.78825794032724,
        "noul": 94.59459459459461,
        "score": 73.93503136595372
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 82.77262796695852
      },
      "llm_router": {
       "accuracy": 0.8817891373801917,
       "cc": 78.90312865239771,
       "cc_by_type": {
        "choice": 82.56581843796776,
        "noul": 97.93814432989691,
        "score": 56.205423189328485
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 78.90312865239771
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -134.40860215053763,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.8888888888888888,
       "cc": 82.2008021480952,
       "cc_by_type": {
        "choice": 87.62886597938144,
        "noul": 100.0,
        "score": 58.973540464904175
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 82.2008021480952
      },
      "multilingual": {
       "accuracy": 0.8436363636363636,
       "cc": 76.30071025864956,
       "cc_by_type": {
        "choice": 78.4805999347897,
        "noul": 75.0,
        "score": 75.42153084115898
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 76.30071025864956
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 74.63974664616975,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 70.58590660517592
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.63974664616975
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 94.1393156079887,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100.0,
        "score": 94.42995883597814
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 94.1393156079887
      },
      "usecase": {
       "accuracy": 0.7636363636363637,
       "cc": 69.35577915701994,
       "cc_by_type": {
        "choice": 70.58823529411767,
        "noul": 75.0,
        "score": 62.47910217694213
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 69.35577915701994
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 81.19799542871031,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100.0,
        "score": 62.64160533375001
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 81.19799542871031
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 79.87151540781498,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100.0,
        "score": 60.35187341238502
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 79.87151540781498
      },
      "de": {
       "accuracy": 0.875,
       "cc": 85.97225175792362,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100.0,
        "score": 81.81056058350534
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.97225175792362
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 65.82742536986764,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 97.6684958488953
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 65.82742536986764
      },
      "en": {
       "accuracy": 0.8757602085143353,
       "cc": 80.5020795009644,
       "cc_by_type": {
        "choice": 82.86572045740228,
        "noul": 85.61151079136691,
        "score": 73.02900725412404
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 80.5020795009644
      },
      "es": {
       "accuracy": 0.9354838709677419,
       "cc": 86.76999083304393,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 70.09790072099149
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 86.76999083304393
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 64.27731458013723,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 92.28549565298
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 64.27731458013723
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 83.31529512153791,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 71.12235595284902
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 83.31529512153791
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 72.25647563538706,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0,
        "score": 16.76942690616119
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 72.25647563538706
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 76.06266465412442,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 42.217612044992116
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 76.06266465412442
      },
      "ja": {
       "accuracy": 0.88,
       "cc": 80.36396885359544,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 100.0,
        "score": 69.68602888008735
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 80.36396885359544
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 59.1851949497821,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 77.55558484934632
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 59.1851949497821
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.42623540510623,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.27870621531868
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.42623540510623
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 45.29308709163208,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 54.49397988961484
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 45.29308709163208
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75.0
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 76.21121333785727,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80.0,
        "score": 56.25107547696585
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 76.21121333785727
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 77.45853245441235,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 74.80341115227651
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 77.45853245441235
      },
      "sv": {
       "accuracy": 0.8181818181818182,
       "cc": 62.003275945801796,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 26.009827837405396
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 62.003275945801796
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 84.17252034238425,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 74.25669146193536
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 84.17252034238425
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 68.25214444086528,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 69.27256235485393
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 68.25214444086528
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 67.66972367558195,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": 95.09956650697184
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 67.66972367558195
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8626666666666667,
       "cc": 82.26142865343769,
       "cc_by_type": {
        "choice": 82.26142865343769
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.26142865343769
      },
      "noul": {
       "accuracy": 0.9226666666666666,
       "cc": 84.53333333333333,
       "cc_by_type": {
        "noul": 84.53333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 84.53333333333333
      },
      "score": {
       "accuracy": 0.8293333333333334,
       "cc": 72.07798903636777,
       "cc_by_type": {
        "score": 72.07798903636777
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 72.07798903636777
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 27.939831762844438,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -25.0,
        "score": 24.81949528853331
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 27.939831762844438
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 77.33853334905363,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100.0,
        "score": 50.64914041983174
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 77.33853334905363
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 82.441425967922,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": 96.10476570864405
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 82.441425967922
      },
      "demand_forecasting": {
       "accuracy": 0.5882352941176471,
       "cc": 60.55897120398711,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 100.0,
        "score": 40.15644577570404
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 60.55897120398711
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8823529411764706,
       "cc": 68.03216403821983,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 70.76315878132617
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 68.03216403821983
      },
      "feature_extraction": {
       "accuracy": 0.8947368421052632,
       "cc": 54.12246561727883,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 66.66666666666667,
        "score": 8.279346537371046
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 54.12246561727883
      },
      "financial_crime": {
       "accuracy": 0.7333333333333333,
       "cc": 61.89527890054794,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 42.85714285714286,
        "score": 52.632615413128406
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 61.89527890054794
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 65.03231166234393,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50.0,
        "score": 76.45582000445339
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 65.03231166234393
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 69.68099890061212,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 100.0,
        "score": 92.3763300351697
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 69.68099890061212
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.94889455125153,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100.0,
        "score": 36.390580870028685
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.94889455125153
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 65.62556455092009,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 77.77777777777777,
        "score": 42.35472982847086
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 65.62556455092009
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 62.45738322463984,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 84.14634322230664
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 62.45738322463984
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.17118595744371,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 67.54618743087235
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.17118595744371
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 77.08049546993229,
       "cc_by_type": {
        "choice": 88.31168831168831,
        "noul": 96.96969696969697,
        "score": 45.960101128411566
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 77.08049546993229
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 80.32276037340623,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100.0,
        "score": 64.00091067875992
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 80.32276037340623
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 55.50483084561898,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 66.60734304753477
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 55.50483084561898
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 78.24962814912611,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100.0,
        "score": 64.30553469368374
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 78.24962814912611
      },
      "scientific_discovery": {
       "accuracy": 0.8260869565217391,
       "cc": 81.73460160704407,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 100.0,
        "score": 74.18931206750902
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 81.73460160704407
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.63219304596917,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 94.5727357215374
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.63219304596917
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.86973938739871,
      92.5387564621637
     ],
     "I": [
      75.18214623594949,
      82.12723110306345
     ],
     "capability": [
      81.40391281836617,
      86.85759269246338
     ],
     "composite": {
      "A": [
       76.93989597679156,
       79.39169870780026
      ],
      "B": [
       76.64350103296658,
       79.9065768646065
      ],
      "C": [
       76.93989597679156,
       79.39169870780026
      ]
     },
     "seed": 75
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.84828412834014,
     "capability": 84.86748441722712,
     "intelligence": 78.8866847061141
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8626666666666667,
      "calibration": 88.413178530724,
      "cc": 82.26142865343769,
      "cc_by_type": {
       "choice": 82.26142865343769
      },
      "cc_tiered": 82.10070579489245,
      "cc_tiered_open": 82.86017015722445,
      "cc_tiered_sealed": 81.34124143256045,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.26142865343769
     },
     "noul": {
      "accuracy": 0.9226666666666666,
      "calibration": 91.26744208624478,
      "cc": 84.53333333333333,
      "cc_by_type": {
       "noul": 84.53333333333333
      },
      "cc_tiered": 84.01742023779886,
      "cc_tiered_open": 84.04347826086956,
      "cc_tiered_sealed": 83.99136221472814,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 84.53333333333333
     },
     "score": {
      "accuracy": 0.8293333333333334,
      "calibration": 92.86423176805167,
      "cc": 72.07798903636777,
      "cc_by_type": {
       "score": 72.07798903636777
      },
      "cc_tiered": 70.541928085651,
      "cc_tiered_open": 69.5259130239514,
      "cc_tiered_sealed": 71.55794314735057,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 72.07798903636777
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.008000000000000007,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 75.07579973698547,
    "Emphasis on Accuracy 60:20:20": 76.55509972242248,
    "Emphasis on Cost 20:20:60": 70.34216842661972,
    "Emphasis on Speed 20:60:20": 78.85873400314371,
    "Intelligence only": 78.8866847061141
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Singularity Compute",
   "axes": {
    "calibration": 91.69553288708067,
    "cost": 56.397623391206686,
    "intelligence": 80.00899811568502,
    "speed": 79.83008200555805
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.035608016166795625,
      "ece_score": 92.87839676664088,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.63072104192983,
      "score_before_time_limit": 92.87839676664088,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.05523784944949169,
      "ece": 0.03925753215871338,
      "n": 375,
      "score": 91.9027642520753,
      "score_before_time_limit": 92.14849356825732,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.9226666666666666,
      "interval90_width": 0.10066666666666665,
      "n": 375,
      "nrps": 0.046295265602298664,
      "score": 90.5531133672369,
      "score_before_time_limit": 90.79523399121348,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.06890002728671593
     }
    },
    "score": 91.69553288708067
   },
   "capability": 85.85226550138285,
   "composite_ci95": {
    "A": [
     73.29668924375432,
     75.30976141875034
    ],
    "B": [
     73.94546816229918,
     76.78676004590665
    ],
    "C": [
     73.29668924375432,
     75.30976141875034
    ]
   },
   "cost": {
    "basis": "Launch list price: the launch list price stated on the vendor's public product page (singularitycompute.com/super-decider, fetched 22 Oct 2026: USD 0.03 per 1M input tokens, output tokens free); the ASI:Cloud catalogue itself was not reachable to us. The page lists one price for the product and no separate rate for thinking mode; we apply it to both rows. Counted as an operator launch list price by the frozen pricing module pricing_v15 (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the launch-list line and the rates for this product are identical in the earlier module 81e3578f), which carries a launch-list-price line for this product. Applied to the tokens reported by the provider's API; not a base-model estimate. Over all 1,500 items the frozen resolve() gives USD 0.052098 per 1,000 decisions; the official cost axis uses the common 1,485-item basis, where it is USD 0.028406 per 1,000 decisions. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 1,406,105 input / 636,221 output tokens. All-item usage 2,604,915 input / 643,964 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05209830000000001
    },
    "kind": "listed",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.028406161616161614
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 4,
    "n_over_limit_answered_ok": 4,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Super Decider (thinking)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid fast-lane evaluation, published at the provider's request (the order was private and was made public at the customer's request). Vendor-hosted, first-party Singularity Compute API (llm.a.singularitycompute.com, POST /v1/systemone, model `super-decider`), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `singularitycompute` (no proxy) before the first request. All 1,500 of 1,500 requests answered with HTTP 200 and the served model was `super-decider` on every row; native probabilities on all three question types; one serial pass, no retries. Run with the provider's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.9137241571549,
    "I_sealed": 81.10427207421513,
    "base": 80.00899811568502,
    "excess": -12.03007468741594,
    "gap": -2.190547917060229,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 89.43664280726739,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 100.0,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 80.56521739130434,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 70.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 66.73931227289297,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 87.8266479316451,
       "hard": 83.83079175879907,
       "judge": 53.50151232669309,
       "standard": 41.86938539100456
      }
     },
     "sealed|choice": {
      "cc": 86.07206252990329,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 79.18353186076095,
       "judge": 87.65915768854063,
       "standard": 91.45749841805527
      }
     },
     "sealed|noul": {
      "cc": 85.33314138254951,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 78.18181818181817,
       "judge": 90.76923076923077,
       "standard": 91.04477611940298
      }
     },
     "sealed|score": {
      "cc": 71.9076123101926,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 88.61860278662654,
       "hard": 70.62401785520835,
       "judge": 70.00494391151904,
       "standard": 68.97330857995443
      }
     }
    }
   },
   "jevbench_score": 74.54050341581339,
   "key": "singularity-super-decider-thinking",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (Hugging Face model card; weights public, not gated); the card states that the licence is the same as that of the base model google/diffusiongemma-26B-A4B-it; the hosted service itself is proprietary",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:llm.a.singularitycompute.com/v1/systemone#super-decider",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9278074866310161,
    "decisive": 374,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1.0,
   "rank": 9,
   "ranked": true,
   "ranks": {
    "A": 9,
    "B": 6,
    "C": 9,
    "capability": 6
   },
   "repo": "https://huggingface.co/Singularity-Compute/super-decider",
   "scores": {
    "A": 74.54050341581339,
    "B": 75.57357140671725,
    "C": 74.54050341581339
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8880362859927118,
    "p50_s_raw": 0.8880362859927118,
    "p95_s_adjusted": 1.171011304546846,
    "p95_s_raw": 1.171011304546846
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no vendor code ran locally. Model, as pinned by the provider: Singularity-Compute/super-decider at 7f8b3f76e88aba6aa1159da75eb577ea990d2af1 on Hugging Face (public, not gated), a discrete-diffusion decision model fine-tuned from google/diffusiongemma-26B-A4B-it and served from an NVFP4 checkpoint (card statement). Serving code, as pinned by the provider: github.com/Singularity-Compute/super-decider at 10bf7eddcbd3b5f6dddf27e868231b2e1140dfba (pinned vLLM nightly with the provider's patches and a structured server). Neither the deployed weights nor the serving stack were independently inspected. MANDATORY DISCLOSURE: measured through the provider's hosted API on 21 Oct 2026; published at the provider's request. THINKING MODE: `\"thinking\": \"always\"` in every request body; the row shows one fixed setting, and the provider's docs say that in \"auto\" mode thinking is skipped when the server is busy, so the default row reflects the server load at the time of measurement (21 Oct 2026, 08:05-08:45 UTC). TRAINING AND SELECTION, provider statement: the public JevBench v1.4/v1.5 items were used as a validation set (recipe and checkpoint choice, serving parameters), not as training data, and the model card says no benchmark test items were used for training; the public set is therefore treated as seen during selection and the normal public-versus-sealed gap rule applies to this row. Provider statements are not independently verified.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 72.35548687610432,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 67.06646062831297
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 72.35548687610432
      },
      "calibration_gen": {
       "accuracy": 0.9603960396039604,
       "cc": 94.04767911079934,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92.0,
        "score": 93.32437349358574
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 94.04767911079934
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 81.46444618137986,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 70.37037037037037,
        "score": 90.2831307753952
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 81.46444618137986
      },
      "fairness": {
       "accuracy": 0.9791666666666666,
       "cc": 94.69056759200323,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 86.66666666666667,
        "score": 97.40503610934303
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 94.69056759200323
      },
      "jaggedness": {
       "accuracy": 0.9277777777777778,
       "cc": 86.73264775638553,
       "cc_by_type": {
        "choice": 91.33782483156881,
        "noul": 94.59459459459461,
        "score": 74.26552384299319
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 86.73264775638553
      },
      "llm_router": {
       "accuracy": 0.8913738019169329,
       "cc": 79.28475475311417,
       "cc_by_type": {
        "choice": 88.1130580258871,
        "noul": 97.93814432989691,
        "score": 51.8030619035585
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 79.28475475311417
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": -27.822335394381014,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": -100.0,
        "score": -6.047651344433369
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.8888888888888888,
       "cc": 83.5588517898443,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 100.0,
        "score": 75.41882341076999
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 83.5588517898443
      },
      "multilingual": {
       "accuracy": 0.8727272727272727,
       "cc": 77.97925523642682,
       "cc_by_type": {
        "choice": 87.28399087055756,
        "noul": 75.0,
        "score": 71.65377483872291
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 77.97925523642682
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 81.25264963526998,
       "cc_by_type": {
        "choice": 90.0,
        "noul": 73.33333333333334,
        "score": 80.42461557247661
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 81.25264963526998
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.984375,
       "cc": 97.86154958999055,
       "cc_by_type": {
        "choice": 96.996996996997,
        "noul": 100.0,
        "score": 96.58765177297464
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 97.86154958999055
      },
      "usecase": {
       "accuracy": 0.7863636363636364,
       "cc": 72.69446270220719,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 83.33333333333333,
        "score": 61.71083908701371
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 72.69446270220719
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 81.37414726403654,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100.0,
        "score": 82.21767988734771
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 81.37414726403654
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 86.74302168543532,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 60.22906505630594
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 86.74302168543532
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 87.86469329901563,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100.0,
        "score": 79.52328343686987
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.86469329901563
      },
      "el": {
       "accuracy": 0.8666666666666667,
       "cc": 82.77739181238248,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 66.66666666666667,
        "score": 98.42528530679364
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 82.77739181238248
      },
      "en": {
       "accuracy": 0.893136403127715,
       "cc": 82.44535092865597,
       "cc_by_type": {
        "choice": 87.03945521777865,
        "noul": 86.33093525179856,
        "score": 73.9656623163907
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 82.44535092865597
      },
      "es": {
       "accuracy": 0.9032258064516129,
       "cc": 80.77921693565571,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 52.12557902882683
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 80.77921693565571
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 87.0450308763467,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100.0,
        "score": 93.92197787494177
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 87.0450308763467
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 81.41948601012723,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 44.258458030381696
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 81.41948601012723
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 67.99732773933682,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100.0,
        "score": 3.9919832180104753
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 67.99732773933682
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 73.99076354729904,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 36.001908724516
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.99076354729904
      },
      "ja": {
       "accuracy": 0.88,
       "cc": 78.37958776791658,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 80.0,
        "score": 69.43582446340027
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 78.37958776791658
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 61.87246167528537,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 85.61738502585612
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 61.87246167528537
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.5452368885293,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.63571066558788
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.5452368885293
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 52.44326408683357,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 49.970484901193366
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 52.44326408683357
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75.0
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 67.40750200911383,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 39.99999999999999,
        "score": 62.222506027341474
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.40750200911383
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 80.38189941531125,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 72.96655858771337
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 80.38189941531125
      },
      "sv": {
       "accuracy": 0.9090909090909091,
       "cc": 74.58559564417607,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 23.756786932528208
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 74.58559564417607
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 84.80472311114137,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 54.41416933342412
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 84.80472311114137
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.77274224161705,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 68.5762912409802
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.77274224161705
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 68.01735999950307,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": 96.14247547873522
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 68.01735999950307
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8986666666666666,
       "cc": 86.91134541418701,
       "cc_by_type": {
        "choice": 86.91134541418701
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 86.91134541418701
      },
      "noul": {
       "accuracy": 0.9253333333333333,
       "cc": 85.06666666666666,
       "cc_by_type": {
        "noul": 85.06666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 85.06666666666666
      },
      "score": {
       "accuracy": 0.8266666666666667,
       "cc": 72.227609837006,
       "cc_by_type": {
        "score": 72.227609837006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 72.227609837006
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 30.4204600919081,
       "cc_by_type": {
        "choice": 68.0,
        "noul": 0.0,
        "score": 23.261380275724306
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 30.4204600919081
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 86.52558672096914,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 59.576760162907405
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 86.52558672096914
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 88.7634187455937,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100.0,
        "score": 82.55041883840713
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 88.7634187455937
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 52.75207268586704,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 100.0,
        "score": 28.43165665409235
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 52.75207268586704
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8823529411764706,
       "cc": 86.10915391772893,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 100.0,
        "score": 72.82021537637524
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 86.10915391772893
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": -4.852470451788558,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 100.0,
        "score": -201.9787950031644
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 53.52947921875822,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 56.106644939187824
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 53.52947921875822
      },
      "gaming": {
       "accuracy": 0.9090909090909091,
       "cc": 88.761530265053,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 100.0,
        "score": 76.73755246763282
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 88.761530265053
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 67.58227437011047,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 100.0,
        "score": 86.08015644366473
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 67.58227437011047
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.89225185440681,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 56.70602037406981
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.89225185440681
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 67.00979355306558,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 77.77777777777777,
        "score": 23.251602881418965
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.00979355306558
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 61.35217022284606,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 80.8307042169253
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 61.35217022284606
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 80.38344987923516,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100.0,
        "score": 64.18297919624672
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 80.38344987923516
      },
      "model_routing": {
       "accuracy": 0.8663594470046083,
       "cc": 75.8636160703572,
       "cc_by_type": {
        "choice": 87.14285714285714,
        "noul": 100.0,
        "score": 40.44799106821445
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.8636160703572
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 88.8673569663905,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 100.0,
        "score": 78.11838567844211
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 88.8673569663905
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 54.1509340689137,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 62.54565271741892
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.1509340689137
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 69.40657921763047,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 100.0,
        "score": 52.55471302234951
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 69.40657921763047
      },
      "scientific_discovery": {
       "accuracy": 0.8695652173913043,
       "cc": 86.73550472317316,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 100.0,
        "score": 74.6992677927079
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 86.73550472317316
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.71909941626893,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.1572982488068
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.71909941626893
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.1809192085414,
      92.77187467791587
     ],
     "I": [
      76.33680965172114,
      83.43172085992364
     ],
     "capability": [
      82.6816915911483,
      87.75097702137747
     ],
     "composite": {
      "A": [
       73.29668924375432,
       75.30976141875034
      ],
      "B": [
       73.94546816229918,
       76.78676004590665
      ],
      "C": [
       73.29668924375432,
       75.30976141875034
      ]
     },
     "seed": 76
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.69553288708067,
     "capability": 85.85226550138285,
     "intelligence": 80.00899811568502
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8986666666666666,
      "calibration": 92.63072104192983,
      "cc": 86.91134541418701,
      "cc_by_type": {
       "choice": 86.91134541418701
      },
      "cc_tiered": 87.75435266858534,
      "cc_tiered_open": 89.43664280726739,
      "cc_tiered_sealed": 86.07206252990329,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 86.91134541418701
     },
     "noul": {
      "accuracy": 0.9253333333333333,
      "calibration": 91.9027642520753,
      "cc": 85.06666666666666,
      "cc_by_type": {
       "noul": 85.06666666666666
      },
      "cc_tiered": 82.94917938692693,
      "cc_tiered_open": 80.56521739130434,
      "cc_tiered_sealed": 85.33314138254951,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 85.06666666666666
     },
     "score": {
      "accuracy": 0.8266666666666667,
      "calibration": 90.5531133672369,
      "cc": 72.227609837006,
      "cc_by_type": {
       "score": 72.227609837006
      },
      "cc_tiered": 69.3234622915428,
      "cc_tiered_open": 66.73931227289297,
      "cc_tiered_sealed": 71.9076123101926,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 72.227609837006
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.002666666666666706,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.16486213201394,
    "Emphasis on Accuracy 60:20:20": 73.7967813820553,
    "Emphasis on Cost 20:20:60": 63.923143914690996,
    "Emphasis on Speed 20:60:20": 73.73581086017921,
    "Intelligence only": 80.00899811568502
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking",
   "axes": {
    "calibration": 77.72284683250058,
    "cost": 48.64218899335634,
    "intelligence": 34.357350203953175,
    "speed": 93.10096025009605
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12117664234817307,
      "ece_score": 75.76467153036539,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 75.76467153036539,
      "score_before_time_limit": 75.76467153036539,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3297587131367292,
      "brier": 0.16945421980960473,
      "ece": 0.10146864036589111,
      "n": 373,
      "score": 79.70627192682178,
      "score_before_time_limit": 79.70627192682178,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.8870967741935484,
      "interval90_width": 0.41084229390681,
      "n": 372,
      "nrps": 0.12225177629429217,
      "score": 77.6975970403146,
      "score_before_time_limit": 77.6975970403146,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.161898141449708
     }
    },
    "score": 77.72284683250058
   },
   "capability": 56.04009851822688,
   "composite_ci95": {
    "A": [
     15.25845206491497,
     31.49102646212039
    ],
    "B": [
     13.201047287610493,
     28.670835708654444
    ],
    "C": [
     10.596147267302063,
     21.868768376472495
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference, the same estimate as the live row standard_one_3b_sh: the model card names base_model StandardThinking/StandardOne-3B, built on mistralai/Ministral-3-3B-Instruct-2512. Neither Standard One checkpoint is listed on OpenRouter; the exact underlying base is priced as OpenRouter mistralai/ministral-3b-2512, list price USD 0.10/M input and USD 0.10/M output (the same OpenRouter document the live sibling cites, read 2026-10-12T00:14Z, source/public-docs/OPENROUTER-MINISTRAL-3-PRICES.json; the frozen module catalogue carries the same 0.10/0.10). Self-hosted weights, no bookable tariff of the measured deployment; the author's stated hosted list price (USD 0.038/M input, standardthinking.ai/pricing) is below this floor, so the base-model reference price applies as a floor. Note on the module: the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026) has no explicit listing decision for this base string (reference() raises), so the price is the registry value 0.10/0.10 carried over from the live entry and verified against that snapshot, not a module lookup. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's own server: input_tokens = 764,986 on the 1,485 answered items, output_tokens = 0 on every row (the schema head generates no text; priced at the reference output price). Our offline count through a first-party transcription of the author's renderer and the pinned tokenizer predicted the server's numbers EXACTLY, item by item and in total (764,986 offline = 764,986 native), and equals the 12 Oct receipt of the v2 row (same tokenizer, template and item set). The 15 refused items (HTTP 422, over the author's 32,768-token context) count 0 tokens.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05099906666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05151420875420875
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 3B SH v4 (Standard Thinking)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.519/h. The authors' reference server code/serve_head.py (127.0.0.1:30171, bf16, torch SDPA, no prefix cache) runs in ONE docker container with network none, driven one request at a time over loopback on the VM by the unchanged pinned pod_driver_v16 (frozen runtime modules sha256-pinned in pre-registration Amendment A1). Requests over the documented 32,768-token context answer HTTP 422 (refused, not truncated). The standard self-hosted x2 + 0.15 s latency adjustment applies. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 07:12-07:19Z; measurement 66 s, total 359 s; scored run 46 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:19Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.78829013651281,
    "I_sealed": 25.92641027139353,
    "base": 34.357350203953175,
    "excess": 7.022353094763574,
    "gap": 16.861879865119285,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 61.56799120865807,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 52.537903757415954,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 30.90614216701173,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 35.89073703386864,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.231557035532845,
       "hard": 37.263336613520615,
       "judge": 44.97516664432012,
       "standard": 15.348483458055373
      }
     },
     "sealed|choice": {
      "cc": 53.65186478422669,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 42.75471261709262,
       "judge": 58.570029382957905,
       "standard": 58.23665893271462
      }
     },
     "sealed|noul": {
      "cc": 2.2708017664270868,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 1.8181818181818077,
       "judge": -13.846153846153841,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 21.85656426352682,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 53.598763156448584,
       "hard": 11.93884335357016,
       "judge": 13.357841341148124,
       "standard": 38.56899102054731
      }
     }
    }
   },
   "jevbench_score": 24.395574588088643,
   "key": "standard_one_3b_sh_v4",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (merged weights, schema head and reference server; base Ministral 3 3B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "StandardThinking/StandardOne-3B-SH@7f56f73be882bc3fb2b69abe609bab1de8297d77",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.84,
    "decisive": 250,
    "decisive_rate": 0.6702412868632708,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 131,
   "ranked": true,
   "ranks": {
    "A": 131,
    "B": 133,
    "C": 149,
    "capability": 179
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-3B-SH",
   "scores": {
    "A": 24.395574588088643,
    "B": 21.82489051004517,
    "C": 16.941371241728227
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21108300800002552,
    "p50_s_raw": 0.030541504000012765,
    "p95_s_adjusted": 0.23198008359992173,
    "p95_s_raw": 0.040990041799960866
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN REFERENCE SERVER, run unmodified, on NEW WEIGHTS: StandardThinking/StandardOne-3B-SH at 7f56f73be882bc3fb2b69abe609bab1de8297d77 (tag v4 = main HEAD on 22 and 23 Oct; MODEL_MANIFEST.json sha256 3071f802ef11e092dfafdd72c49a616b8a288c7a0791e0966f6f2fd55021a863, 26 files, 7,976,303,844 weight bytes, 8 chart PNGs excluded and never read; 5 LFS oids and 21 git-blob oids checked, the author's SHA256SUMS agrees on all 24 non-PNG entries; all 26 files re-verified on the VM after download). Command (the card's Quick-start, all defaults): python3 /code/code/serve_head.py --snapshot /models/StandardOne-3B-SH --head /models/StandardOne-3B-SH/head --port 30171 --served-model-name standardthinking/standard-one-3b-sh (batch 8 requests / 32,768 tokens / 5 ms wait, bf16 backbone, torch SDPA, no cache). The 18 files executed or read at inference (LICENSE, SYSTEM_PROMPT.txt, chat_template.jinja, code/{head,model,render,scoring,serve_head}.py, code/requirements.txt, config.json, generation_config.json, model.safetensors.index.json, params.json, processor_config.json, special_tokens_map.json, tekken.json, tokenizer.json, tokenizer_config.json) are byte-identical (sha256) to the v2 route that was security-reviewed (PR on file / PR on file) and measured on 12 Oct 2026; only the two backbone weight shards, head/schema_head.safetensors and head/schema_head_config.json differ (the config diff is two provenance keys, step 57 -> 11471 and snapshot -> StandardOne-3B-SH@v3, which the server never reads: scoring.py:162-170). The source changes and runner were reviewed before this run; the unchanged serving code retained its earlier clearance. Image: pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime@sha256:eee11b3b3872a8c838e35ef48f08b2d5def2080902c7f666831310ca1a0ef2be plus the hash-pinned runtime supplement (19 wheels), built offline on the VM (image sha256:6d8e3c0234ba9923d2c59b85c6b046e6438a34dc1d1504a0218f9a3d00e59f12; the build is not bit-reproducible, the 12 Oct image was 28f68ce6...); transformers 5.12.1, peft 0.21.0, tokenizers 0.22.2, torch 2.11.0, Python 3.12. Container read-only, network none, model and code read-only. Weights: Apache-2.0, public, not gated, tensors in safetensors only (no pickle). FIRST MEASUREMENT OF THIS VERSION. NEW VERSION OF AN EXISTING MODEL (Author submission, free queue, filed by the authors as a follow-up to, not a replacement of, the live row standard_one_3b_sh, which measured the v2 weights at revision bead20e94b7c96045a19dc4d43af48a66c8911ea via paid order 3a081599 on 12 Oct 2026). Whether this row supersedes the live row or stands beside it is a release-owner decision. AUTHOR'S OWN SERVER, on our hardware: the live row ran on a Lium H100; this row runs on one RTX PRO 6000 (bf16 SDPA numerics can differ across GPUs by tiny amounts), so the two rows differ in hardware as well as in weights. SECOND USE OF THE DRAW: evaluator exposure and device ruling admitted this as a same-author second use of the frozen draw for the key family standard_one_3b_sh (first use: the 12 Oct v2 row), with this disclosure; any further StandardThinking row goes to the next draw. 10 s RULE: max measured decision latency 0.116 s (p50 0.031 s, p95 0.041 s, p99 0.102 s); 0 decisions over 10 s. The 15 refused items are the 15 longest (76k-82k tokens), refused with HTTP 422 `too_long` at positions 65, 169, 474, 482, 515, 573, 580, 729, 833, 899, 951, 1057, 1116, 1198, 1418 exactly as pre-registered and identical to the 12 Oct set; they are scored wrong, never truncated. DETERMINISM: first 50 positions run twice, 2/50 bit-identical, 0 argmax flips, max probability difference 9.2e-08. AUTHOR EXPOSURE AND DIRECTION (author's own release-manifest.json, self-reported, not verified): the v4 head was trained from the unsubmitted v3 (head config snapshot StandardOne-3B-SH@v3); between v2 and v4 the author reports its internal `jevbench-original` set falling 98.61 -> 95.83 and `hardproxy` 41.67 -> 38.0 while `internal-eval-v1` rises 84.33 -> 86.33 and `-v2-realistic` 88.33 -> 90.17. The author's training data and overlap screening for v3/v4 were not independently audited by us, and we ran no overlap check against the public 231. OFFLINE COUNTER: a first-party transcription (no author import) that reproduced the 12 Oct native counts exactly; it does not transcribe the span-coverage checks; the exact sealed token match with the 12 Oct receipt establishes the accounting on this draw, not full renderer equivalence. No outside request for this model, and no score, rank or expectation was communicated to the submitter. PREDECESSOR: standard_one_3b_sh (/jev-models/standard_one_3b_sh); both measurements remain separate rows.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 30.43862198739271,
       "cc_by_type": {
        "choice": 50.0,
        "noul": 50.0,
        "score": -8.684134037821867
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 30.43862198739271
      },
      "calibration_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 38.96843819494644,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 43.99999999999999,
        "score": 20.6253570026548
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 38.96843819494644
      },
      "composite_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 24.719858196839443,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -11.111111111111116,
        "score": 17.791010904881468
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 24.719858196839443
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 54.95165466462041,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 44.854963993861254
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 54.95165466462041
      },
      "jaggedness": {
       "accuracy": 0.4388888888888889,
       "cc": 10.357653869679638,
       "cc_by_type": {
        "choice": 27.81520692974015,
        "noul": -35.13513513513513,
        "score": 38.39288981443389
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 10.357653869679638
      },
      "llm_router": {
       "accuracy": 0.7284345047923323,
       "cc": 56.86247915081065,
       "cc_by_type": {
        "choice": 66.71656247248393,
        "noul": 60.82474226804124,
        "score": 43.04613271190677
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.86247915081065
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -31.17577443919728,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75.0,
        "score": -19.55825115264338
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6,
       "cc": 29.820965381161074,
       "cc_by_type": {
        "choice": 60.873818063254,
        "noul": -2.777777777777779,
        "score": 31.366855858007003
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 29.820965381161074
      },
      "negation_pair_gen": {
       "accuracy": 0.6395348837209303,
       "cc": 21.33088783112437,
       "cc_by_type": {
        "choice": 80.0,
        "noul": -19.999999999999996,
        "score": 3.9926634933731164
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 21.33088783112437
      },
      "option_order": {
       "accuracy": 0.5625,
       "cc": 44.074567243675105,
       "cc_by_type": {
        "choice": 44.074567243675105
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 44.074567243675105
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 75.75773444552337,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100.0,
        "score": 63.30923937260613
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 75.75773444552337
      },
      "usecase": {
       "accuracy": 0.4954545454545455,
       "cc": 18.505383917285155,
       "cc_by_type": {
        "choice": 54.65686274509807,
        "noul": -16.666666666666664,
        "score": 17.525955673424054
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 18.505383917285155
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -16.223038433749565,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 50.0,
        "score": -84.3834010155344
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 40.03490559353603,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -13.228616552725248
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 40.03490559353603
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 68.64915139458252,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 54.841259493482085
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.64915139458252
      },
      "el": {
       "accuracy": 0.4,
       "cc": 19.79738156682247,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 93.09791751238548
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 19.79738156682247
      },
      "en": {
       "accuracy": 0.5899218071242398,
       "cc": 31.93655338456402,
       "cc_by_type": {
        "choice": 54.30858788640606,
        "noul": 14.388489208633093,
        "score": 27.112583058652916
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.93655338456402
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 46.78575129260497,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 50.0,
        "score": 19.721038543394044
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 46.78575129260497
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -7.626126671344963,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100.0,
        "score": 75.48227572367003
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.0
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 43.697632308903025,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0.0,
        "score": 52.26936751494436
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 43.697632308903025
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 25.716200068649442,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25.0,
        "score": 35.668153278574025
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 25.716200068649442
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.9707899465536,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100.0,
        "score": 10.971606004898549
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.9707899465536
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 23.91155193522305,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 0.0,
        "score": 14.625839284620701
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 23.91155193522305
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -15.139117321498192,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": -0.9729075200501258
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 68.65378847904408,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 89.29469877046562
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 68.65378847904408
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 18.275963913872197,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 14.135251048975883
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.275963913872197
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 26.945577097442794,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -19.999999999999996,
        "score": 31.306473145904345
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 26.945577097442794
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 62.57546899861372,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100.0,
        "score": 30.154220784880657
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.57546899861372
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 10.245504472118656,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 52.864172990824045
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 10.245504472118656
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": -4.064699344436785,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": -68.71583716374514
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 67.03729233509661,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 65.62800603754792
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.03729233509661
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 18.87224921870604,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -51.29285686365588
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.87224921870604
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6573333333333333,
       "cc": 55.739681203237716,
       "cc_by_type": {
        "choice": 55.739681203237716
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 55.739681203237716
      },
      "noul": {
       "accuracy": 0.56,
       "cc": 12.00000000000001,
       "cc_by_type": {
        "noul": 12.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 12.00000000000001
      },
      "score": {
       "accuracy": 0.5066666666666667,
       "cc": 27.031098893672045,
       "cc_by_type": {
        "score": 27.031098893672045
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.031098893672045
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -11.271742464625499,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -75.0,
        "score": -26.815227393876494
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 7.5198762513336215,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -14.017320866427019
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 7.5198762513336215
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 49.10493423741544,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 29.83512791549836
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 49.10493423741544
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -44.25518595295127,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100.0,
        "score": -50.89421282961406
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 14.797957491146107,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 6.712713053148455
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 14.797957491146107
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -21.042436730888507,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -79.47951145052716
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 3.4881564022506595,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": 21.10872690983322
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 3.4881564022506595
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -2.2430386640931927,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50.0,
        "score": 16.441615715037493
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -11.621822353601146,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": -12.365467060803436
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.0
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": 6.876774725855042,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 0.0,
        "score": 23.414050087629366
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 6.876774725855042
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 28.644604238960103,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -33.333333333333336,
        "score": 42.52296000370202
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 28.644604238960103
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 11.417587720502413,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": -34.564441139568025
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.417587720502413
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 7.739621289484592,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60.0,
        "score": 29.28412298553631
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 7.739621289484592
      },
      "model_routing": {
       "accuracy": 0.7235023041474654,
       "cc": 54.23699976619281,
       "cc_by_type": {
        "choice": 73.11688311688313,
        "noul": 60.606060606060595,
        "score": 28.988055575634696
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 54.23699976619281
      },
      "moderation_trust_safety": {
       "accuracy": 0.3888888888888889,
       "cc": -5.564829778188596,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 29.37076978251675
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 47.02903163973576,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 24.46685350787955
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 47.02903163973576
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -8.643300724068723,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100.0,
        "score": 18.405073197251966
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -20.59884799463212,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -66.66666666666667,
        "score": -8.173355578099262
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0.0
      },
      "search_retrieval": {
       "accuracy": 0.7916666666666666,
       "cc": 63.30212811247312,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 50.0,
        "score": 61.25869750467913
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 63.30212811247312
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.01156479992845,
      80.83105889189729
     ],
     "I": [
      28.310092387743147,
      38.15335637133141
     ],
     "capability": [
      51.20193899458882,
      58.72712563492504
     ],
     "composite": {
      "A": [
       15.25845206491497,
       31.49102646212039
      ],
      "B": [
       13.201047287610493,
       28.670835708654444
      ],
      "C": [
       10.596147267302063,
       21.868768376472495
      ]
     },
     "seed": 73
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.72284683250058,
     "capability": 56.04009851822688,
     "intelligence": 34.357350203953175
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6573333333333333,
      "calibration": 75.76467153036539,
      "cc": 55.739681203237716,
      "cc_by_type": {
       "choice": 55.739681203237716
      },
      "cc_tiered": 57.60992799644238,
      "cc_tiered_open": 61.56799120865807,
      "cc_tiered_sealed": 53.65186478422669,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.739681203237716
     },
     "noul": {
      "accuracy": 0.56,
      "calibration": 79.70627192682178,
      "cc": 12.00000000000001,
      "cc_by_type": {
       "noul": 12.00000000000001
      },
      "cc_tiered": 16.588471966719407,
      "cc_tiered_open": 30.90614216701173,
      "cc_tiered_sealed": 2.2708017664270868,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 12.00000000000001
     },
     "score": {
      "accuracy": 0.5066666666666667,
      "calibration": 77.6975970403146,
      "cc": 27.031098893672045,
      "cc_by_type": {
       "score": 27.031098893672045
      },
      "cc_tiered": 28.87365064869773,
      "cc_tiered_open": 35.89073703386864,
      "cc_tiered_sealed": 21.85656426352682,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.031098893672045
     }
    },
    "raw_on_A": {
     "calibration": 83.50326227116736,
     "capability": 58.44228421245579,
     "intelligence": 33.38130615374423
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.19385254275724,
    "Emphasis on Accuracy 60:20:20": 18.836883828876903,
    "Emphasis on Cost 20:20:60": 22.008806459889172,
    "Emphasis on Speed 20:60:20": 27.28596975238759,
    "Intelligence only": 15.353422081196133
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "matthewagi (Hugging Face)",
   "axes": {
    "calibration": 88.71404111864382,
    "cost": 42.09847950547972,
    "intelligence": 44.07459281825939,
    "speed": 90.72909335832355
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06361399351573381,
      "ece_score": 87.27720129685324,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 87.27720129685324,
      "score_before_time_limit": 87.27720129685324,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.3726541554959786,
      "brier": 0.11603929170212504,
      "ece": 0.07014905306792772,
      "n": 373,
      "score": 85.97018938641445,
      "score_before_time_limit": 85.97018938641445,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.45833333333333365,
      "n": 372,
      "nrps": 0.08941220418134077,
      "score": 92.89473267266376,
      "score_before_time_limit": 92.89473267266376,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.026346571182691923
     }
    },
    "score": 88.71404111864382
   },
   "capability": 66.3943169684516,
   "composite_ci95": {
    "A": [
     22.90103740348847,
     39.617459397512
    ],
    "B": [
     21.037394977458085,
     37.86516816679001
    ],
    "C": [
     15.903498196866996,
     27.51212458160555
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7): base string Qwen/Qwen3.6-35B-A3B (the card's base_model, confirmed by the author's export.json) priced as qwen/qwen3.6-35b-a3b USD 0.15/M input, USD 1.0/M output. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the author server's own usage block (input_tokens = len(input_ids) of the encoded request: state plus all schema and template tokens; output_tokens = 0, one forward pass with a decision head, nothing generated). The offline count through a first-party transcription of the author's encoder and the pinned tokenizer predicted the server's numbers EXACTLY, item by item: 842,725 input tokens over the 1,485 answered items, and 15 HTTP 422 refusals at the pre-registered ids and positions; the 15 refused items count 0 tokens. OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.085124 per 1,000 decisions on the common basis (USD 0.084273 over all 1,500 items) is 1.318x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 and v1.6.24 imajev-9b precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them); their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0842725
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.08512373737373737
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Tern-35B-A3B (matthewagi, Qwen3.6 35B-A3B MoE decision model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.525/h. The author's own `python3 -m tern serve` from the pinned commit on 127.0.0.1:8000 (vLLM backend on 127.0.0.1:8101) inside a network-none Docker container with the cut proven from inside (loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only weights, harness and task input, empty credential placeholders, HF offline flags, one request at a time (no concurrency, no batching endpoint). Timed over loopback on the VM. One disclosed deviation: a read-write compile-cache bind shared between the synthetic smoke container and the scored container, so the second cold start fits the time limit; it carries compiler artifacts only and leaves content on the pod disk, which is deleted. Measured 22 Oct 2026 (VM reserved 06:29Z, removed 06:48Z) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked locally, on the node and inside the container); determinism 50/50 bit-identical; node shredded and VM destroyed afterwards (teardown confirmed).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 50.428471552943506,
    "I_sealed": 37.720714083575274,
    "base": 44.07459281825939,
    "excess": 2.8682306990125213,
    "gap": 12.707757469368232,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 76.60705387746165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 72.31377719182598,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 28.017253278122844,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 46.661107503246036,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.43405451873939,
       "hard": 48.41853406073319,
       "judge": 43.25118143648162,
       "standard": 46.874669980671655
      }
     },
     "sealed|choice": {
      "cc": 63.47155961797452,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 57.32624031455995,
       "judge": 58.570029382957905,
       "standard": 72.47416156928918
      }
     },
     "sealed|noul": {
      "cc": 13.087561949116248,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 7.272727272727275,
       "judge": 1.538461538461533,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 36.603020683635066,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 58.76270966326589,
       "hard": 26.38182260397771,
       "judge": 39.45655585509179,
       "standard": 41.68526959594928
      }
     }
    }
   },
   "jevbench_score": 32.05539762294736,
   "key": "tern-35b-a3b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (HF card metadata and model LICENSE; the Qwen base licence is retained as BASE_MODEL_LICENSE; code repo Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "matthewagi/Tern-35B-A3B@ef783c939186c1363f1de46d63564e78bb22f1c5",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9572649572649573,
    "decisive": 234,
    "decisive_rate": 0.6273458445040214,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 109,
   "ranked": true,
   "ranks": {
    "A": 109,
    "B": 109,
    "C": 112,
    "capability": 111
   },
   "repo": "https://huggingface.co/matthewagi/Tern-35B-A3B",
   "scores": {
    "A": 32.05539762294736,
    "B": 30.125368419778187,
    "C": 22.260692793713446
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2874846070000331,
    "p50_s_raw": 0.06874230350001653,
    "p95_s_adjusted": 0.2940871632998323,
    "p95_s_raw": 0.07204358164991617
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own `tern` package (Matthew-agi/Tern at d154fa085d92eb2170681cd90817e18c735553ff, tree 529c3f40c3cc966b15b3e964a5c79ea9bde6c894, Apache-2.0, 66 files, every git blob oid verified; no tag exists on that repo; the README-tested commit 9723b745 differs only in README/docs), unmodified, installed from the pinned tree into our own image and started as `python3 -m tern serve --profile rtx-pro-6000 --model <pinned verified snapshot> --cache-dir /cache/tern --gpu 0 --host 127.0.0.1 --port 8000 --backend-port 8101` (the author's Docker command plus our local --model). Author profile kept unchanged: max_model_len 32,768, 32,768 batch tokens, 256 seqs, CUDA-graph capture 16,384, BF16, Triton recurrent kernels, head batch 16/16,384, CUBLAS_WORKSPACE_CONFIG=:4096:8, CLEF_SUBMISSION_MODE=single, all 40 layers and 8 experts, no result cache; the h100, h100-bf16 and a100 profiles and the HF runtime/serve.py were not used. Static source review PASS WITH FINDINGS, no blockers (no non-loopback network, no telemetry, no item logging, no pickle/trust_remote_code, no rule keyed to item text or benchmark names). DISCLOSED CODE PATTERNS, cleared by the benchmark owner (21 Oct 2026; author code on a rented Verda pod only, weights pre-staged, network none): (1) the author's default start downloads the 33 model files from the Hub (src/tern/artifacts.py:36-56, called from launch.py:105) - never reached, because --model points to a pre-staged local snapshot (artifacts.py:37-38 returns before any Hub import), with HF offline flags, empty credentials and a network-none container; (2) a second snapshot_download in an unused research helper (kev_vllm/clef_schema.py:501-507, no caller); (3) an exec-based rewrite of one vLLM function (kev_vllm/expert_act_quant.py:75), off in the RTX BF16 profile; (4) vLLM monkey-patches behind the author's own sha256 check of 16 vLLM source files (fail-closed); (5) subprocess calls (nvidia-smi and the server's own two child processes, list argv, no shell; src/tern/launch.py:25,33,122) inside the network-none container. Weights: matthewagi/Tern-35B-A3B at ef783c939186c1363f1de46d63564e78bb22f1c5, exactly the author's 33 inference files (69,778,082,886 bytes), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file on the host, then re-hashed by the author's own verify() at every container start. Image: our own build on the pod (network on, before any sealed byte existed) from the author's digest-pinned base nvidia/cuda:12.9.1-devel-ubuntu24.04 plus the author's 196 pins re-expressed with published sha256 hashes (--require-hashes) and the pinned source tree; torch 2.13.0+cu129, vllm 0.30.0+cu129, transformers 5.19.0, triton 3.7.1, flashinfer 0.6.18.post1; image id sha256:c097beb417b5ea0d34ea5a5c144a7159d42e79dad8e9067847cce00f9e5e7ae1. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry 56, Hugging Face decision-index tag). AUTHOR-DEFAULT CONTEXT CAP: the author's server answers HTTP 422 natively above its 32,768-token limit (whole state plus all questions and options, strict >, no truncation; serve_clef_api.py:59-60,149-150); no status proxy was used. The 15 items of 76k-81k tokens (76,424-80,587) were refused exactly as pre-registered by id and position and are scored wrong, like the other self-hosted rows on this draw; the largest answered item is 1,886 tokens and none is within 64 tokens of the limit. SERIAL ONLY: the author states that answers shift with server load and batching (the Decision Index lab found 98.39% agreement with the submitted rows on 827 rows against its 98.5% bar, which it attributes to 128 concurrent requests in the submitted run versus one at a time in the check; listed under a maintainer decision); we sent strictly one request at a time, and determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). The probabilities need not equal the author's or the DI lab's (author: cross-GPU and cross-batch differences are unproven equivalent). TRAINING DATA, the author's own statement (TRAINING_DISCLOSURE.md, unverified): full-depth Qwen3.6-35B-A3B backbone with merged adapters (checkpoint 1674, about 500M tokens) and a jointly trained decision head, trained on 52 source families that include the teacher-derived jev_distill and open_jev data and train splits of public benchmarks (for example BANKING77, CLINC, HoVer, GSM8K, When2Call); 150 Decision Index-derived BBH, MMLU-Pro and When2Call requests were observed during development; the author's overlap audit lists 3,238 candidate requests with shared text among the 140,178 public Decision Index requests, 1,392 of them in index benchmarks (not proven identical decisions), and states this is not an uncontaminated held-out claim. Our own check of the exact published text hashes against our public 231 items found 0 hits; it cannot see teacher-derived knowledge or unpublished training text. The author flags 8,678 ANLI and 4,304 ContractNLI training records tagged non-commercial in intermediate dataset metadata (author: not a conclusion about the weights); the model is licensed Apache-2.0 and not marked non-commercial. SELF-REPORTED, unverified: public Decision Index score 59.01 on RTX PRO 6000, warmed serial latency median 59.48 ms on the author's card. 10 s RULE: max measured decision latency 0.092 s over all 1,500 rows; 0 decisions over 10 s.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 74.08010601583455,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": 55.573651380836964
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 74.08010601583455
      },
      "calibration_gen": {
       "accuracy": 0.45544554455445546,
       "cc": 14.58961955883486,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 4.0000000000000036,
        "score": 3.3955819002585486
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.58961955883486
      },
      "composite_gen": {
       "accuracy": 0.7128712871287128,
       "cc": 42.81849470657943,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -3.703703703703709,
        "score": 48.419350425068
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 42.81849470657943
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.10507525772684,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 55.31522577318052
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.10507525772684
      },
      "jaggedness": {
       "accuracy": 0.5277777777777778,
       "cc": 22.74275699647896,
       "cc_by_type": {
        "choice": 43.69586140519731,
        "noul": -13.513513513513509,
        "score": 38.04592309775307
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 22.74275699647896
      },
      "llm_router": {
       "accuracy": 0.7667731629392971,
       "cc": 61.80122015484502,
       "cc_by_type": {
        "choice": 71.47133926212908,
        "noul": 58.762886597938135,
        "score": 55.169434604467824
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 61.80122015484502
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": -9.903319954898492,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75.0,
        "score": -5.225423782221261
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6545454545454545,
       "cc": 37.785915212351604,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": -2.777777777777779,
        "score": 46.45831441581074
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 37.785915212351604
      },
      "negation_pair_gen": {
       "accuracy": 0.7093023255813954,
       "cc": 32.55812757855488,
       "cc_by_type": {
        "choice": 96.66666666666667,
        "noul": -19.999999999999996,
        "score": 21.007716068997983
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 32.55812757855488
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 78.40789816963463,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100.0,
        "score": 59.24771853292792
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.40789816963463
      },
      "usecase": {
       "accuracy": 0.65,
       "cc": 42.58590768435422,
       "cc_by_type": {
        "choice": 69.36274509803921,
        "noul": 16.666666666666675,
        "score": 41.728311288356764
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 42.58590768435422
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 18.21433405809204,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": -57.26175973048579
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 18.21433405809204
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 60.8667161853777,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 49.26681522279978
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 60.8667161853777
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 75.73588042373903,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 76.10144658095163
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 75.73588042373903
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.37421355089219,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 97.82841346459463
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.37421355089219
      },
      "en": {
       "accuracy": 0.6759339704604691,
       "cc": 43.172496055235456,
       "cc_by_type": {
        "choice": 66.82979216753516,
        "noul": 23.741007194244613,
        "score": 38.9466888039266
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 43.172496055235456
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 66.70348822602944,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 9.898392899948005
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.70348822602944
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 42.8880393396527,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 93.69144042332968
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 42.8880393396527
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 85.8572915694662,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100.0,
        "score": 78.74834529663389
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 85.8572915694662
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 32.83579577600802,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25.0,
        "score": 40.26716386433691
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 32.83579577600802
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 55.92058462532583,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -18.208628041403642
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 55.92058462532583
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 13.024478812283887,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 0.0,
        "score": 10.558742235104246
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 13.024478812283887
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -11.465428872083232,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 10.048157828194759
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0.0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 70.13123827831198,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 93.7270481682693
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 70.13123827831198
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -7.542437762022018,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -22.627313286066062
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 35.69444255134223,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 42.318198580814645
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 35.69444255134223
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 58.23344183481286,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 46.521185846218174
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 58.23344183481286
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": 3.902465116747834,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60.0,
        "score": 73.83505492471159
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 3.902465116747834
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 14.384514212851292,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0.0,
        "score": -35.10732692666352
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 14.384514212851292
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.65884973589452,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100.0,
        "score": 82.4926782399416
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.65884973589452
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 25.3227144997153,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -33.333333333333336,
        "score": 9.301476832479239
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 25.3227144997153
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.748,
       "cc": 67.45058267475459,
       "cc_by_type": {
        "choice": 67.45058267475459
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 67.45058267475459
      },
      "noul": {
       "accuracy": 0.5973333333333334,
       "cc": 19.466666666666676,
       "cc_by_type": {
        "noul": 19.466666666666676
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 19.466666666666676
      },
      "score": {
       "accuracy": 0.5946666666666667,
       "cc": 39.24988739460201,
       "cc_by_type": {
        "score": 39.24988739460201
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 39.24988739460201
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -8.428579510134812,
       "cc_by_type": {
        "choice": 52.0,
        "noul": -75.0,
        "score": -2.2857385304044353
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.0
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 26.598713535860508,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 11.111111111111116,
        "score": 5.952110241812014
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 26.598713535860508
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 38.90421156137979,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50.0,
        "score": -17.027202714234612
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 38.90421156137979
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -33.96164054317912,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -31.709483033046126
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 18.741946383761825,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 33.03743335418403
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 18.741946383761825
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -12.090688196852398,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": -106.71231616288424
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 37.27941422899224,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 55.535721678573346
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 37.27941422899224
      },
      "gaming": {
       "accuracy": 0.5454545454545454,
       "cc": 4.196787496841249,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -100.0,
        "score": 43.949247507945344
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 4.196787496841249
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 28.96044993083234,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 48.54801645916369
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 28.96044993083234
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 43.26818788361002,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 47.863093272528864
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 43.26818788361002
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 50.09458538469488,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 28.578329797495496
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 50.09458538469488
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -28.18153018272051,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100.0,
        "score": -20.028461515903473
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 33.99841601057772,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 33.511562811003785
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 33.99841601057772
      },
      "model_routing": {
       "accuracy": 0.7880184331797235,
       "cc": 63.33652101229035,
       "cc_by_type": {
        "choice": 81.2987012987013,
        "noul": 66.66666666666667,
        "score": 42.04419507150307
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 63.33652101229035
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 3.642456085476842,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100.0,
        "score": 56.99262737351306
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.642456085476842
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 56.69479476307428,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 53.46414287789509
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.69479476307428
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 17.422823881920966,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50.0,
        "score": 31.82512189206831
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 17.422823881920966
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 3.548736057529934,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 35.28388933201009
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 3.548736057529934
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 77.87106760875567,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": 83.61320282626701
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.87106760875567
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.9737930322135,
      88.87866305138917
     ],
     "I": [
      38.25178041426043,
      48.41158630707976
     ],
     "capability": [
      61.60504955133157,
      67.74481906925115
     ],
     "composite": {
      "A": [
       22.90103740348847,
       39.617459397512
      ],
      "B": [
       21.037394977458085,
       37.86516816679001
      ],
      "C": [
       15.903498196866996,
       27.51212458160555
      ]
     },
     "seed": 79
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.71404111864382,
     "capability": 66.3943169684516,
     "intelligence": 44.07459281825939
    },
    "per_type": {
     "choice": {
      "accuracy": 0.748,
      "calibration": 87.27720129685324,
      "cc": 67.45058267475459,
      "cc_by_type": {
       "choice": 67.45058267475459
      },
      "cc_tiered": 70.03930674771809,
      "cc_tiered_open": 76.60705387746165,
      "cc_tiered_sealed": 63.47155961797452,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.45058267475459
     },
     "noul": {
      "accuracy": 0.5973333333333334,
      "calibration": 85.97018938641445,
      "cc": 19.466666666666676,
      "cc_by_type": {
       "noul": 19.466666666666676
      },
      "cc_tiered": 20.552407613619547,
      "cc_tiered_open": 28.017253278122844,
      "cc_tiered_sealed": 13.087561949116248,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 19.466666666666676
     },
     "score": {
      "accuracy": 0.5946666666666667,
      "calibration": 92.89473267266376,
      "cc": 39.24988739460201,
      "cc_by_type": {
       "score": 39.24988739460201
      },
      "cc_tiered": 41.63206409344055,
      "cc_tiered_open": 46.661107503246036,
      "cc_tiered_sealed": 36.603020683635066,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.24988739460201
     }
    },
    "raw_on_A": {
     "calibration": 83.4083275232536,
     "capability": 65.09795629748626,
     "intelligence": 46.78758507171892
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 28.757515138652213,
    "Emphasis on Accuracy 60:20:20": 26.781111146588213,
    "Emphasis on Cost 20:20:60": 26.237683442839387,
    "Emphasis on Speed 20:60:20": 34.6408090395176,
    "Intelligence only": 24.278269607215453
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "scar-ai (Hugging Face)",
   "axes": {
    "calibration": 91.31679956749362,
    "cost": 29.91580590336227,
    "intelligence": 67.78675651753129,
    "speed": 88.01635603332511
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04184131147541006,
      "ece_score": 91.631737704918,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.631737704918,
      "score_before_time_limit": 91.631737704918,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.1769436997319035,
      "brier": 0.07210259847184984,
      "ece": 0.047123056300268036,
      "n": 373,
      "score": 90.57538873994639,
      "score_before_time_limit": 90.57538873994639,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.978494623655914,
      "interval90_width": 0.3006272401433691,
      "n": 372,
      "nrps": 0.050566275277777725,
      "score": 91.7432722576165,
      "score_before_time_limit": 91.7432722576165,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05728413978494613
     }
    },
    "score": 91.31679956749362
   },
   "capability": 79.55177804251245,
   "composite_ci95": {
    "A": [
     19.908037346263423,
     20.507148564469063
    ],
    "B": [
     20.368386686922726,
     21.344865958051614
    ],
    "C": [
     19.908037346263423,
     20.507148564469063
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): exact base string Qwen/Qwen3.8-27B resolves directly to the listing USD 0.42/M input, USD 3.00/M output (dated 25 Sep 2026 catalog snapshots); applied here as qwen/qwen3.8-27b USD 0.42/M input, USD 0.0/M output, i.e. with the output priced at 0 under the pricing ruling on file. No explicit_ref override is needed. Estimated, not charged, not a vendor tariff or a quote from the author or any host (release-owner ruling on file). Tokens as reported by the usage block of the author's server: input_tokens counts the rendered question prompt, including chat template and options; output_tokens is the author's positive estimate sum(len(json.dumps(answer))//4) of the answer JSON; nothing is generated (pooling runner). Under the zero-output convention for native-probability rows (the zero-output pricing convention) the output is priced at USD 0/M; the server-reported estimate of 84,061 output tokens is disclosed but not billed; billing that estimate at USD 3.00/M would give USD 0.386660 per 1,000 decisions on the common basis = 5.99x the cap. Measured totals: 766,685 input and 84,061 estimated output tokens across the 1,485 answered items. Input usage equals the pre-registered first-party offline count item by item, offset 0 and 0 mismatches; this agreement does not turn the output estimate into a tokenizer count. The 15 refused HTTP 400 items carry no usage and count 0 tokens; these are the 15 standard long positions excluded from every row's cost on the common 1,485-item basis (v1.6.30), so common-basis totals are also 766,685 input and 84,061 estimated output tokens. COST ON THE COMMON BASIS: USD 0.216840 per 1,000 decisions on 1,485 items, 3.357 times the official cost cap USD 0.064595; estimated, not charged. OVER THE OFFICIAL COST CAP, DISCLOSED: ranked_over_cap; outside the official-caps views.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.2146718
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.21684020202020202
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vera Large",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner. Attempt 2 measured 24 Oct 2026 08:36-08:53Z, about USD 0.74 GPU rental (lead-reported approximation); teardown confirmed 08:53:47Z. Author server on 127.0.0.1:8200 in a --network none --read-only container; serial loopback driver. Read-only code, model, input and exported cache; empty credential variables, HF offline flags and author steering switches unset. Cut proof inside the container before held-out input: only loopback, no IPv4/IPv6 default route, DNS and TCP fail. Complete=true, deadline_hit=false, rc 0, measurement container 308 s (run cap 538 s), scored loop 155.156841 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked inside the container. Raw output sha256 468809896014afa52e577106142dbf1ff21d4d75ee7b55fe02d1f5993ee8fcea (receipt only; not opened by this drafting worker). Final node-shred verified; exact-id teardown confirms provider absent and guard closed. GPU rental is separate from the benchmark's estimated base-model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.66290812065732,
    "I_sealed": 62.91060491440525,
    "base": 67.78675651753129,
    "excess": -0.0872235641036383,
    "gap": 9.752303206252073,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 83.61812666623719,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 88.13447593935399,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 69.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50.0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 64.4223575301033,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.83060606060607,
       "hard": 78.76416666666667,
       "judge": 60.168644067796606,
       "standard": 39.91518518518517
      }
     },
     "sealed|choice": {
      "cc": 79.73075848398948,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 75.5406499363941,
       "judge": 77.08129285014691,
       "standard": 85.76249736342545
      }
     },
     "sealed|noul": {
      "cc": 49.20962825131636,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 39.99999999999999,
       "judge": 44.61538461538461,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 59.791428007909914,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 81.4221052631579,
       "hard": 55.378788732394376,
       "judge": 56.79755208333334,
       "standard": 62.2921818181818
      }
     }
    }
   },
   "jevbench_score": 20.313436479396994,
   "key": "vera-large",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 in the model card and vera-s1 0.1.6 package metadata; no standalone LICENSE file; public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "scar-ai/vera-large@cc5adb16238d9ebae887da71f994cd04963d251f",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9543973941368078,
    "decisive": 307,
    "decisive_rate": 0.8230563002680965,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 147,
   "ranked": true,
   "ranks": {
    "A": 147,
    "B": 143,
    "C": 129,
    "capability": 33
   },
   "repo": "https://huggingface.co/scar-ai/vera-large",
   "scores": {
    "A": 20.313436479396994,
    "B": 20.997537887808015,
    "C": 20.313436479396994
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3349350960000038,
    "p50_s_raw": 0.0924675480000019,
    "p95_s_adjusted": 0.4714153714000302,
    "p95_s_raw": 0.16070768570001512
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER: scar-ai/vera-large at cc5adb16238d9ebae887da71f994cd04963d251f with the author's vera-s1 0.1.6 package, installed from the repo's bundled wheel (sha256 94f5a09cf70e4480994230300963789841e0a7de3ff424b2df619fd76fe4de46) with --no-deps --no-index under a network-free image build; all members equal the PyPI 0.1.6 wheel's (different zip timestamps). Six author Python modules; the model repo's separate Python files and fp32 branch were not fetched. Author files are unchanged; DISCLOSED SERVING WRAPPER: a one-line wrapper around vera.load passes max_num_seqs=256 through the author's own **engine_kw API, then runs vera.serve as __main__, with --model /model --engine vllm --max-model-len 32768 --host 127.0.0.1 --port 8200. This caps the scheduler's maximum co-batched sequences, following the pre-sealed attempt-1 startup failure; it does not change the author's 32,768-token limit, prompt rendering or refusal set. vLLM 0.30.0 pooling runner (token_embed), bidirectional full attention (is_causal: false), causal Gated-DeltaNet layers, prefix caching and chunked prefill off, max_num_batched_tokens = max_model_len = 32768, author-default gpu_memory_utilization 0.9, fp32 option-pointer readout. One request at a time. Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 plus the one wheel, derived image sha256:9ab6ff921642238843ef1daa85e6c05cf38c938cb7f776e4c84a12bcd6d6b61c. torch 2.13.0, transformers 5.17.0, tokenizers 0.23.2, safetensors 0.8.0. The author's export_backbone transformed the pinned backbone offline before any held-out input, then the cache was mounted read-only. Export verification: 850 bf16 backbone tensors, names/shapes/dtypes checked, 43 tensors byte-compared by sampling, every exported file hashed, verdict OK; not a byte comparison of every tensor. Weights: 25,628,527,110 parameters (25,624,600,064 bf16 backbone + 3,927,046 fp32 heads), 10 shards totalling 51,265,014,328 bytes, 16 data files in the pinned manifest (sha256 57d88387c7503633362b61eb9c17d05ea8608accf48774ab13cf68f51634cdde). The submitted code and serving wrapper were reviewed and cleared before the run; evaluator exposure and the device were cleared for both attempts. The source, runner, device selection change and serving wrapper changes were reviewed before the run; required safeguards were checked and the code and device cleared. DISCLOSED PATTERNS: dormant Hub downloads and Hub-capable loaders, weights-only torch.load fallbacks absent from the safetensors serving path, steering environment switches asserted unset, and the contained pre-input cache-writing export. No subprocess, outbound HTTP, telemetry, credential reads or request-body logging in the reviewed author package; offline flags and the network cut were checked inside the measurement container. FIRST MEASUREMENT of Vera Large in this intake. AUTHOR'S OWN SERVER on our rented hardware, vLLM pooling path with the disclosed max_num_seqs=256 wrapper; author files unchanged. NATIVE OUTPUTS: choice and score answers with probabilities, and noul P(true), from the author's readout and option pointer; probabilities rounded to 4 decimals; no generated explanation. AUTHOR EXPOSURE: training and contamination statements in the card are author claims; no independent overlap or contamination audit was run by us. The card's Decision Index 68.4 on a 5% sample and 139 ms median on an AMD MI300X are self-reported and do not transfer to this measurement. LICENCE NOTE: Apache-2.0 in the model card and package metadata; no standalone LICENSE file. ATTEMPT HISTORY: attempt 1 (pod window 24 Oct 2026 06:22-06:32Z, about USD 0.45) failed BEFORE any held-out item, at vLLM startup. CUDA-graph memory profiling tried to allocate a 78,920,024,064-byte (78.9 GB) throwaway cache of 512 sequences alongside about 51 GB of weights. pre-registration Amendment 3 documents the fix: max_num_seqs=256 through vera.load(**engine_kw), reducing that profiling allocation to 39,460,012,032 bytes; author files, the 32,768-token limit and the refusal set unchanged. The runner was rechecked, and code, evaluator exposure and device clearance preceded attempt 2, this measurement. NOT ANSWERED, counted wrong: exactly the 15 pre-registered long items exceed the author's 32,768-token limit (HTTP 400), at positions in the pre-registered long-item set; never truncated. Answered: 1,485 of 1,500 items sent, HTTP 200; all observed refusal ids and positions match the preregistration, no unexpected failures. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 0.294317 s (median 0.092944 s, p95 0.161959 s) over the 1,500 requests. DETERMINISM: 50 positions repeated, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0, on the server's rounded outputs. CONTEXT: author limit 32,768 tokens; no truncation. OVER COST CAP: the base-model estimate (input priced, output at 0 under the zero-output convention, the pricing ruling on file) is above the cost cap, so the row is ranked_over_cap (uncapped views only); the exact figure and ratio are in the cost basis.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 69.1088888888889,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100.0,
        "score": 15.660000000000029
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 69.1088888888889
      },
      "calibration_gen": {
       "accuracy": 0.6534653465346535,
       "cc": 45.135582140332275,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 28.000000000000004,
        "score": 58.308125
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 45.135582140332275
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 87.7554431010122,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 73.9257692307692
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 87.7554431010122
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.34037037037037,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 90.02111111111111
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.34037037037037
      },
      "jaggedness": {
       "accuracy": 0.7111111111111111,
       "cc": 49.24603884811041,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 13.513513513513509,
        "score": 67.42960784313726
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.24603884811041
      },
      "llm_router": {
       "accuracy": 0.9329073482428115,
       "cc": 84.61928322280761,
       "cc_by_type": {
        "choice": 96.03768600862904,
        "noul": 95.8762886597938,
        "score": 61.943875000000006
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.61928322280761
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 33.14086892488955,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 0.0,
        "score": 48.90714285714287
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 33.14086892488955
      },
      "multilingual": {
       "accuracy": 0.7672727272727272,
       "cc": 59.326481019702726,
       "cc_by_type": {
        "choice": 79.45875448320835,
        "noul": 36.111111111111114,
        "score": 62.40957746478871
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 59.326481019702726
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 74.49422222222223,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 73.33333333333334,
        "score": 56.816
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.49422222222223
      },
      "option_order": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100.0
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 79.16758373758374,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100.0,
        "score": 55.52076923076924
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.16758373758374
      },
      "usecase": {
       "accuracy": 0.7181818181818181,
       "cc": 54.12956209150327,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 25.0,
        "score": 60.673
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.12956209150327
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 45.03722832722832,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50.0,
        "score": 23.20692307692307
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 45.03722832722832
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 42.36644444444443,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": -6.234000000000028
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 42.36644444444443
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 80.36392458637937,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100.0,
        "score": 80.91478260869566
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 80.36392458637937
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.478482309124765,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0.0,
        "score": 99.95500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.478482309124765
      },
      "en": {
       "accuracy": 0.8140747176368376,
       "cc": 68.59329552513215,
       "cc_by_type": {
        "choice": 81.98703945521775,
        "noul": 60.431654676258994,
        "score": 63.36119244391971
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 68.59329552513215
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.27265137702716,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 29.60588235294116
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.27265137702716
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.14881602914389,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 97.89999999999998
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.14881602914389
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 95.69543859649123,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 87.08631578947367
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 95.69543859649123
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 48.9343156424581,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25.0,
        "score": 55.322500000000005
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 48.9343156424581
      },
      "id": {
       "accuracy": 1.0,
       "cc": 100.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 87.97631778198418,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 77.95857142857145
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 87.97631778198418
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 38.191514782455215,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 19.999999999999996,
        "score": 23.16866666666668
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.191514782455215
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 24.918809523809518,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": 74.75642857142856
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 24.918809523809518
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.8872222222222,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 97.32833333333333
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.8872222222222
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 0.7546580086580121,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100.0,
        "score": 28.237999999999996
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.7546580086580121
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50.0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 57.96418817886866,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 19.999999999999996,
        "score": 61.509999999999984
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 57.96418817886866
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 80.66579447602031,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100.0,
        "score": 63.21129032258066
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 80.66579447602031
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 40.81872340425533,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 13.520000000000021
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.81872340425533
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 64.54112318840579,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": 15.362500000000011
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 64.54112318840579
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.98397849462366,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 93.21
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.98397849462366
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 45.65444444444446,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 3.630000000000022
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 45.65444444444446
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.856,
       "cc": 81.40033295700262,
       "cc_by_type": {
        "choice": 81.40033295700262
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.40033295700262
      },
      "noul": {
       "accuracy": 0.7813333333333333,
       "cc": 56.266666666666666,
       "cc_by_type": {
        "noul": 56.266666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 56.266666666666666
      },
      "score": {
       "accuracy": 0.728,
       "cc": 62.235198555956686,
       "cc_by_type": {
        "score": 62.235198555956686
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 62.235198555956686
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 17.159411764705876,
       "cc_by_type": {
        "choice": 84.0,
        "noul": -50.0,
        "score": 17.478235294117628
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 17.159411764705876
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 76.62366539257845,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100.0,
        "score": 67.13807692307694
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 76.62366539257845
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 41.596558265582665,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 7.3100000000000165
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 41.596558265582665
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -26.405562865497085,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100.0,
        "score": -9.041250000000023
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.86371980676329,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 84.91
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.86371980676329
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 45.7403668763103,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": 29.045000000000076
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 45.7403668763103
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 46.06900883430294,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 53.333076923076916
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 46.06900883430294
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 51.76553451497248,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 0.0,
        "score": 65.74956521739131
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 51.76553451497248
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 13.172037037037038,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60.0,
        "score": 62.016111111111115
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 13.172037037037038
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 78.19744576384834,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 80.77363636363637
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 78.19744576384834
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 60.99109427609428,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 55.55555555555556,
        "score": 27.417727272727287
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 60.99109427609428
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -0.41153673835126386,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -37.793750000000024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 55.56472222222222,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 46.69416666666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.56472222222222
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 80.70044180645743,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 49.806953125
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.70044180645743
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 18.187698995145087,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100.0,
        "score": 66.07941176470588
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 18.187698995145087
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 34.23259683832108,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 69.45730769230771
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 34.23259683832108
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 30.910332270839376,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0.0,
        "score": 22.287647058823534
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.910332270839376
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 30.59180193236715,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 68.587
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 30.59180193236715
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.88903508771931,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 96.6671052631579
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.88903508771931
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.13496634601603,
      92.25078661376436
     ],
     "I": [
      62.7370903412337,
      71.4169057679515
     ],
     "capability": [
      75.79878347672776,
      81.10866180633397
     ],
     "composite": {
      "A": [
       19.908037346263423,
       20.507148564469063
      ],
      "B": [
       20.368386686922726,
       21.344865958051614
      ],
      "C": [
       19.908037346263423,
       20.507148564469063
      ]
     },
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.31679956749362,
     "capability": 79.55177804251245,
     "intelligence": 67.78675651753129
    },
    "per_type": {
     "choice": {
      "accuracy": 0.856,
      "calibration": 91.631737704918,
      "cc": 81.40033295700262,
      "cc_by_type": {
       "choice": 81.40033295700262
      },
      "cc_tiered": 81.67444257511333,
      "cc_tiered_open": 83.61812666623719,
      "cc_tiered_sealed": 79.73075848398948,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.40033295700262
     },
     "noul": {
      "accuracy": 0.7813333333333333,
      "calibration": 90.57538873994639,
      "cc": 56.266666666666666,
      "cc_by_type": {
       "noul": 56.266666666666666
      },
      "cc_tiered": 59.57893420847392,
      "cc_tiered_open": 69.94824016563147,
      "cc_tiered_sealed": 49.20962825131636,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.266666666666666
     },
     "score": {
      "accuracy": 0.728,
      "calibration": 91.7432722576165,
      "cc": 62.235198555956686,
      "cc_by_type": {
       "score": 62.235198555956686
      },
      "cc_tiered": 62.106892769006606,
      "cc_tiered_open": 64.4223575301033,
      "cc_tiered_sealed": 59.791428007909914,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.235198555956686
     }
    },
    "raw_on_A": {
     "calibration": 88.88233246209342,
     "capability": 78.24991223853809,
     "intelligence": 67.61749201498274
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.037148848749926,
    "Emphasis on Accuracy 60:20:20": 20.101171587311665,
    "Emphasis on Cost 20:20:60": 14.161235731991471,
    "Emphasis on Speed 20:60:20": 21.75816701438723,
    "Intelligence only": 24.266450679341077
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 86.75010672241689,
    "cost": 64.22331428461467,
    "intelligence": 50.78521313506569,
    "speed": 93.06198600681341
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.058056764953574355,
      "ece_score": 88.38864700928512,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.21013171582798,
      "score_before_time_limit": 88.38864700928512,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.22666666666666668,
      "brier": 0.11893480874288055,
      "ece": 0.06490408428587044,
      "n": 375,
      "score": 86.55508083273084,
      "score_before_time_limit": 87.01918314282591,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.936,
      "interval90_width": 0.36888888888888893,
      "n": 375,
      "nrps": 0.08737047233013917,
      "score": 86.48510761869188,
      "score_before_time_limit": 87.18256816400391,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.08448908219489133
     }
    },
    "score": 86.75010672241689
   },
   "capability": 68.7676599287413,
   "composite_ci95": {
    "A": [
     52.46857067195836,
     71.26924673239367
    ],
    "B": [
     47.89198630140442,
     67.28432104483169
    ],
    "C": [
     36.4365074110822,
     60.00705945839956
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string Qwen/Qwen3.5-4B priced as Qwen/Qwen3.5-4B USD 0.03/M input, USD 0.15/M output, as required by the frozen pricing policy. The module selects the deprecated-after-cutoff DeepInfra reference retained under the frozen policy (price snapshots of 25 Sep 2026), USD 0.03/M input and USD 0.15/M output. Estimated, not charged; not a quote from the author or any host. Tokens as reported by the author's server: input usage includes the rendered question and options; output_tokens = 0 because nothing is generated. Reported usage is 1,951,205 input tokens and 0 output tokens, equal to the evaluator's offline count, offset 0 and 0 item-level mismatches. The 15 standard long items contribute 1,180,015 input tokens. Excluding them as for every row on the common 1,485-item cost basis (v1.6.30) leaves 771,190 input tokens and 0 output tokens. All 1,500 items still count for Intelligence. Measured arithmetic: USD 0.015579596 per 1,000 decisions on the common basis = 0.241190172x the unchanged USD 0.06459465517241379 cap; separately USD 0.039024100/1,000 over all 1,500 items = 0.604138220x. GPU rental is separate from this estimated model tariff. COST ON THE COMMON BASIS: USD 0.015580 per 1,000 decisions on 1,485 items, 0.241 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0390241
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01557959595959596
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "vLLM-SR d3-mini",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB, FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner (300 GB disk). Same GPU class as the published d3 and d3-turbo rows; hardware/mode conditions differ across the wider field. The author's own unmodified stdlib server on loopback in a --network none --read-only container; serial requests, read-only code/model/input, empty credential variables and HF offline flags. In-container network cut proven before sealed input; all 22 snapshot files re-hashed, offline flag values and absence of author-path override variables checked. The standard self-hosted x2 + 0.15 s own-server adjustment is an assumption, not measured. Measured 24 Oct 2026, pod window 09:07-09:19Z, approximately USD 0.50 rental. Measurement container 307 s, run cap 2856 s, complete=true, deadline_hit=false, rc 0; scored loop 224.870576 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 814d4afdcc638f3773e3b6263f98e1319e952a2115e6ff1f3cca6c18e98f88af. Node-shred and row cleanup verified; exact-id VM and volume teardown recorded, provider absent-or-discontinued and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 56.122863162304306,
    "I_sealed": 45.44756310782708,
    "base": 50.78521313506569,
    "excess": 0.8357732841215153,
    "gap": 10.675300054477226,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 70.89077514236276,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 66.02705253224286,
       "judge": 68.35860250494397,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 46.39406487232574,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 50.0,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 51.083749472224426,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.81334321079525,
       "hard": 50.57315275281009,
       "judge": 50.10563825926111,
       "standard": 50.70731286121263
      }
     },
     "sealed|choice": {
      "cc": 63.956026815917554,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 57.32624031455995,
       "judge": 62.09598432908915,
       "standard": 74.3724952541658
      }
     },
     "sealed|noul": {
      "cc": 31.726758059233617,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 21.818181818181827,
       "judge": 19.999999999999996,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 40.65990444833008,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 58.88216493509184,
       "hard": 41.7254986343324,
       "judge": 30.53343502273288,
       "standard": 44.607289971340336
      }
     }
    }
   },
   "jevbench_score": 69.52370884635917,
   "key": "vllm-sr-d3-mini",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card, LICENSE and NOTICE); base Qwen/Qwen3.5-4B Apache-2.0; public, not gated. Training data not disclosed",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/d3-mini@bd0090dd758d32ba59ef5c749910a8e87fc4391f",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9097222222222222,
    "decisive": 288,
    "decisive_rate": 0.7721179624664879,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 32,
   "ranked": true,
   "ranks": {
    "A": 32,
    "B": 48,
    "C": 57,
    "capability": 100
   },
   "repo": "https://huggingface.co/vllm-sr/d3-mini",
   "scores": {
    "A": 69.52370884635917,
    "B": 64.74578789680288,
    "C": 49.80867515810295
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20653101700005436,
    "p50_s_raw": 0.02826550850002718,
    "p95_s_adjusted": 0.2392302618999281,
    "p95_s_raw": 0.044615130949964055
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN ENGINE, run unmodified: upstream vllm-sr/d3-mini at bd0090dd758d32ba59ef5c749910a8e87fc4391f, the author's stdlib d3_server.py with d3_runtime.py, d3_format.py, d3_engine.py, d3_fast.py and d3_kernels.py. The measured repository is vllm-sr/d3-mini; zils-ai/z3-mini-v1 and zils-ai/d3-mini-4.5b-v1 are byte-identical weight mirrors, not separate measurements. The server, engine and kernels are byte-identical to the reviewed d3-turbo pin 689b3af198baaf6f11110be38a9daa7188dfd8ad; d3_runtime.py and d3_format.py are the older non-causal-only variants, reviewed separately. The d3-mini weight shards/readout share 0 LFS sha256 objects with the measured d3 and d3-turbo siblings (r25-card/sibling-weight-distinct-20261024.txt). Author defaults: --batch-size 8, --verify fast, warm-up on, device cuda:0, no input-length cap; no evaluator launcher, shim, patch or flag changes the model path. bf16 Qwen3.5-4B backbone, noncausal_full_attention (8 full-attention layers bidirectional, 24 Gated-DeltaNet layers causal), SDPA, plain CUDA / PyTorch path; the author's ROCm fast path does not apply on CUDA. fp32 255-way readout, temperature 1.0, permutation_average off, one forward pass per question, no generation. Python stack: torch 2.13.0, transformers 5.17.0, tokenizers 0.23.2, safetensors 0.8.0, accelerate 1.15.0, fla-core 0.5.2 and flash-linear-attention 0.5.2. Base image vllm/vllm-openai:v0.31.0@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b; derived image sha256:c22d99ab6aca5d4f947d311da6af754a1d17b9183da92e85cd8a39e66c7fa057, hash-pinned wheel overlay installed offline. vLLM is in the image, not the serving backend. All 22 pinned snapshot files hash-checked in the measuring container; author MODEL_MANIFEST.json sha256 bc638e17982411f6ad0f524d33d0dc1f0db81ad7995c1bfdf93dba7674df5e48. The submitted code was reviewed and cleared at the exact pin; evaluator exposure, the device, pricing and context settings were cleared before the run. The source was reviewed with required safeguards; the runner was reviewed and findings fixed, with cold nonce probes and a maximum-of-repetitions fit check applied before the final review. pre-registration-D3MINI.md Amendments 1 and 2, including the token correction, govern this run. Weights: 4,539,265,536 bf16 backbone parameters plus fp32 readout; two weight shards plus readout.safetensors total 9,081,227,360 bytes. FIRST MEASUREMENT of vLLM-SR d3-mini (our own intake). AUTHOR'S OWN ENGINE on our rented hardware, one request at a time; the unchanged pinned pod_driver_v16 and TypeSafeAdapter drive the author's server. NATIVE OUTPUTS: choice, noul and score probabilities; no generated answer text. AUTHOR EXPOSURE: training data not disclosed by the card; no overlap or contamination audit was run by us. The author's Decision Index figures are a different measurement and are not independently verified by us. CONTEXT: as shipped, no input-length cap (decision_config.json max_length null and /health max_input_tokens null), no lower evaluator-imposed cap and no truncation; all 15 standard long items attempted, as required by ruling on file point 4. This differs from the published d3-turbo row, measured with its author's 32,768-token default cap; the published d3 row also had no cap. All 1,500 items answered HTTP 200, 0 refusals, preregistered class A_all_answered. 10 s RULE: exactly 15 decisions over 10 s, the standard long items at positions in the pre-registered long-item set, taking 10.543-17.503 s; these count wrong in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 17.503080 s, median 0.028272 s, p95 0.044697 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. This is a distinct checkpoint from d3 and d3-turbo, with no shared LFS weight shards/readout; the zils-ai mirrors are not additional entries.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 40.63164823450948,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100.0,
        "score": -44.771721963138255
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 40.63164823450948
      },
      "calibration_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 37.73069621287146,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": 43.99999999999999,
        "score": 29.637475701180648
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 37.73069621287146
      },
      "composite_gen": {
       "accuracy": 0.8316831683168316,
       "cc": 73.75553872088415,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 70.37037037037037,
        "score": 67.15640839390811
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 73.75553872088415
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 63.10908224458594,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 19.999999999999996,
        "score": 69.32724673375782
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 63.10908224458594
      },
      "jaggedness": {
       "accuracy": 0.55,
       "cc": 28.69703799210934,
       "cc_by_type": {
        "choice": 39.364773820981725,
        "noul": 13.513513513513509,
        "score": 33.21282664183279
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 28.69703799210934
      },
      "llm_router": {
       "accuracy": 0.7955271565495208,
       "cc": 67.03168558806748,
       "cc_by_type": {
        "choice": 75.43365325350004,
        "noul": 71.13402061855669,
        "score": 54.5273828921457
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 67.03168558806748
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -9.927812416090854,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75.0,
        "score": 19.443366875438784
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.0
      },
      "multilingual": {
       "accuracy": 0.6836363636363636,
       "cc": 45.17330606603309,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 16.666666666666675,
        "score": 50.15419708082939
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 45.17330606603309
      },
      "negation_pair_gen": {
       "accuracy": 0.8255813953488372,
       "cc": 61.06174933224694,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 46.66666666666666,
        "score": 43.18524799674084
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 61.06174933224694
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 84.510651461271,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100.0,
        "score": 68.54696939882801
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.510651461271
      },
      "usecase": {
       "accuracy": 0.6,
       "cc": 40.20280485542287,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 20.833333333333325,
        "score": 38.99076750744508
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 40.20280485542287
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -9.311607822425833,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -50.0,
        "score": -20.79196632442035
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0.0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 35.36820231523675,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": -6.491399198683001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 35.36820231523675
      },
      "de": {
       "accuracy": 0.75,
       "cc": 73.36610796233718,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 100.0,
        "score": 67.88593450648057
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 73.36610796233718
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 27.681387194716347,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 99.99015785975425
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 27.681387194716347
      },
      "en": {
       "accuracy": 0.6993918331885317,
       "cc": 51.59869853180825,
       "cc_by_type": {
        "choice": 65.95111116535065,
        "noul": 45.32374100719425,
        "score": 43.52124342287985
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 51.59869853180825
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 62.08157234583805,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 5.820573481233538
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.08157234583805
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 44.736663698553684,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 66.45042825413101
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 44.736663698553684
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 77.01958842781792,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 73.41170645992435
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 77.01958842781792
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 29.535686556230626,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -25.0,
        "score": 63.886389277630414
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 29.535686556230626
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 80.31764827211875,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 54.9825628989751
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 80.31764827211875
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 36.165394585019165,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 11.38736723400906
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.165394585019165
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 1.193668607293527,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100.0,
        "score": 48.02545026632503
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 1.193668607293527
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.36440108052369,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 95.75986990823773
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.36440108052369
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -2.357836041615627,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0,
        "score": -7.073508124846883
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0.0
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 56.11967176332876,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 63.593886216774244
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.11967176332876
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 56.672476568725976,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 52.44524349521742
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 56.672476568725976
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 48.98508227773069,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 52.48716172680909
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 48.98508227773069
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 42.022185067124546,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0,
        "score": -23.933444798626358
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 42.022185067124546
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.61883570307816,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 41.114571625363524
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.61883570307816
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 47.94274705325116,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 10.494907826420164
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 47.94274705325116
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.74,
       "cc": 66.41726783903252,
       "cc_by_type": {
        "choice": 66.41726783903252
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 66.41726783903252
      },
      "noul": {
       "accuracy": 0.6986666666666667,
       "cc": 39.733333333333334,
       "cc_by_type": {
        "noul": 39.733333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 39.733333333333334
      },
      "score": {
       "accuracy": 0.6053333333333333,
       "cc": 44.100372801134455,
       "cc_by_type": {
        "score": 44.100372801134455
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 44.100372801134455
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 6.389036030080741,
       "cc_by_type": {
        "choice": 68.0,
        "noul": -50.0,
        "score": 1.1671080902422193
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.389036030080741
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 14.76164364876596,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 7.707981325869994
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 14.76164364876596
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 39.51825430650474,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0.0,
        "score": 67.33525072439225
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 39.51825430650474
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -27.57346684962377,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": -67.5157221862982
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 21.618848047320437,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 41.66813834485987
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 21.618848047320437
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -56.86789857623726,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": -241.04394730103883
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 38.30068301713463,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 20.22417790294452
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 38.30068301713463
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 10.160005486264879,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50.0,
        "score": 32.74482482116397
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 10.160005486264879
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 2.534002247445445,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60.0,
        "score": 50.935340075669664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 2.534002247445445
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 42.47689290860803,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 45.48920834752287
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 42.47689290860803
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 28.530781387905918,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -11.111111111111116,
        "score": 19.959269228317243
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 28.530781387905918
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 12.886497100776092,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 34.35841603351108
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.886497100776092
      },
      "llm_guardrails": {
       "accuracy": 0.7037037037037037,
       "cc": 55.97192219635028,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 47.915766589050826
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.97192219635028
      },
      "model_routing": {
       "accuracy": 0.8433179723502304,
       "cc": 70.69713565968036,
       "cc_by_type": {
        "choice": 91.81818181818183,
        "noul": 78.78787878787878,
        "score": 41.485346372980445
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 70.69713565968036
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 11.277283989001177,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100.0,
        "score": 56.864481525544775
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 11.277283989001177
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 56.91434168592918,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 54.122783646459794
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.91434168592918
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 18.561851789482514,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50.0,
        "score": 20.46388049160025
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 18.561851789482514
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 10.043399462512967,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 35.927299836814264
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 10.043399462512967
      },
      "search_retrieval": {
       "accuracy": 1.0,
       "cc": 98.39989987588321,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 95.19969962764965
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.39989987588321
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.15681201246277,
      87.71391054836684
     ],
     "I": [
      44.60747334361465,
      55.13127592033528
     ],
     "capability": [
      63.96092710139239,
      70.47357791075639
     ],
     "composite": {
      "A": [
       52.46857067195836,
       71.26924673239367
      ],
      "B": [
       47.89198630140442,
       67.28432104483169
      ],
      "C": [
       36.4365074110822,
       60.00705945839956
      ]
     },
     "seed": 84
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.75010672241689,
     "capability": 68.7676599287413,
     "intelligence": 50.78521313506569
    },
    "per_type": {
     "choice": {
      "accuracy": 0.74,
      "calibration": 87.21013171582798,
      "cc": 66.41726783903252,
      "cc_by_type": {
       "choice": 66.41726783903252
      },
      "cc_tiered": 67.42340097914015,
      "cc_tiered_open": 70.89077514236276,
      "cc_tiered_sealed": 63.956026815917554,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.41726783903252
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 86.55508083273084,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 39.06041146577968,
      "cc_tiered_open": 46.39406487232574,
      "cc_tiered_sealed": 31.726758059233617,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.6053333333333333,
      "calibration": 86.48510761869188,
      "cc": 44.100372801134455,
      "cc_by_type": {
       "score": 44.100372801134455
      },
      "cc_tiered": 45.871826960277254,
      "cc_tiered_open": 51.083749472224426,
      "cc_tiered_sealed": 40.65990444833008,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 44.100372801134455
     }
    },
    "raw_on_A": {
     "calibration": 87.76116280589144,
     "capability": 69.60593042154665,
     "intelligence": 51.45069803720186
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.20751583797981,
    "Emphasis on Accuracy 60:20:20": 58.55588872032517,
    "Emphasis on Cost 20:20:60": 64.81023745341724,
    "Emphasis on Speed 20:60:20": 74.07624833806008,
    "Intelligence only": 50.78521313506569
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "org2ai (Hugging Face)",
   "axes": {
    "calibration": 88.05577650422971,
    "cost": 49.579289707036324,
    "intelligence": 65.50854607947286,
    "speed": 92.01872463451872
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0364505263232707,
      "ece_score": 92.70989473534586,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.70989473534586,
      "score_before_time_limit": 92.70989473534586,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.12762808226321673,
      "ece": 0.09810736388535705,
      "n": 375,
      "score": 80.37852722292858,
      "score_before_time_limit": 80.37852722292858,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.9813333333333333,
      "interval90_width": 0.44444444444444464,
      "n": 375,
      "nrps": 0.07497524415840688,
      "score": 91.07890755441471,
      "score_before_time_limit": 91.07890755441471,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.05172330237664942
     }
    },
    "score": 88.05577650422971
   },
   "capability": 76.78216129185128,
   "composite_ci95": {
    "A": [
     66.33206507388527,
     68.99172007480622
    ],
    "B": [
     64.97009367669243,
     68.7071684124402
    ],
    "C": [
     66.33206507388527,
     68.99172007480622
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string google/gemma-4-26b-a4b-it (the card's base_model google/gemma-4-26B-A4B-it in the lower-case spelling that the module lists; the exact card spelling is in none of its decision tables and raises) priced as google/gemma-4-26b-a4b-it USD 0.09/M input, USD 0.3/M output. The same estimate on the same base is already carried by the published rows respan-span-02 and shisa-de-1. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's reader: input_tokens = the tokens of the one read per question (system prompt, state, options and chat template; no item has more than 26 options, so there is one read each and no knockout); output_tokens = 0 on every row (gate 0, nothing generated). Over all 1,500 items the server reported 1,982,239 input tokens. PRE-REGISTERED USAGE PREDICTION MISSED BY A CONSTANT +1: our offline count through a first-party transcription of the author's prompt path and the pinned tokenizer predicted 1,980,739 tokens (exact equality, item by item), and the server's usage.input_tokens was our count plus one on every one of the 1,500 items. This MISSES the pre-registration (exact equality, stop and report): the runner carried a +-2 token tolerance that was not pre-registered, and the +1 was already visible on all six smoke anchors before the first sealed byte, so the run did not stop (lead Amendment 1c in the pre-registration records this). The offset is constant and systematic (probably a leading BOS token added by the server; that reading is ours and was not verified); its cause is unknown; billing uses the server's usage like every other row. 0 refusals, as pre-registered. Cost effect of the +1: 0.19 % (USD 0.047939 vs 0.047850 per 1,000 on the common basis), no rank change in the scoring field. The 15 very long items (76,550-81,725 tokens) are answered in full and count for no row's cost on the common 1,485-item cost basis (v1.6.30). COST ON THE COMMON BASIS: USD 0.047939 per 1,000 decisions on 1,485 items, 0.742 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.11893433999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04793915151515151
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Wald 26B-A4B (Gemma 4 26B-A4B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. Stock vLLM 0.30.0 (127.0.0.1:8371) and the author's reader eval.systemone_vllm (127.0.0.1:8000) run in ONE docker container with network none; the reader was timed over loopback, one request at a time. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 06:50-07:07Z; measurement 338 s, total 981 s; scored run 204 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:07Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 71.5000217659513,
    "I_sealed": 59.51707039299441,
    "base": 65.50854607947286,
    "excess": 2.143424602601179,
    "gap": 11.98295137295689,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 73.59023844561248,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 72.31377719182598,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 83.56521739130434,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 73.91304347826086,
       "judge": 80.0,
       "standard": 100.0
      }
     },
     "open|score": {
      "cc": 57.34460946093709,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.66872201243804,
       "hard": 55.94711638549248,
       "judge": 62.13648005986813,
       "standard": 54.7897334376793
      }
     },
     "sealed|choice": {
      "cc": 75.24186878564784,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 69.81612119810339,
       "judge": 73.55533790401567,
       "standard": 82.91499683611053
      }
     },
     "sealed|noul": {
      "cc": 56.17462002742497,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 47.272727272727266,
       "judge": 53.846153846153854,
       "standard": 64.17910447761194
      }
     },
     "sealed|score": {
      "cc": 47.134722365910434,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 65.87205357327875,
       "hard": 48.67115104064259,
       "judge": 37.1986896099809,
       "standard": 49.597248546656246
      }
     }
    }
   },
   "jevbench_score": 68.21122197345876,
   "key": "wald-26b-a4b",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (model card and Hub metadata; base google/gemma-4-26B-A4B-it: apache-2.0 in its card metadata with a link to the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "org2ai/Wald-26B-A4B@4a5d0b94bff26a8bba359ef311b3e535cefae901",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.824,
    "decisive": 375,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 41,
   "ranked": true,
   "ranks": {
    "A": 41,
    "B": 34,
    "C": 34,
    "capability": 50
   },
   "repo": "https://huggingface.co/org2ai/Wald-26B-A4B",
   "scores": {
    "A": 68.21122197345876,
    "B": 67.4156737392911,
    "C": 68.21122197345876
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2362664810000524,
    "p50_s_raw": 0.043133240500026204,
    "p95_s_adjusted": 0.26590433870004515,
    "p95_s_raw": 0.057952169350022585
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN READER, run unmodified, in front of stock vLLM. Author code: 10 Python modules of org2ai/Wald-26B-A4B at 4a5d0b94bff26a8bba359ef311b3e535cefae901 (eval/__init__.py, eval/systemone_vllm.py, eval/vllm_letter.py, kev/__init__.py, kev/api.py, midtrain/__init__.py, midtrain/encode.py, midtrain/letter.py, midtrain/rawprompt.py, midtrain/tokens.py; CODE_MANIFEST.json sha256 5ce8feb796d52add3dfa4520fa0fe418c4eaaf4551591ad7d2485aca72aefaa8), hash-verified, mounted read-only at /code. The unshipped modules midtrain/chat_lora.py (contains snapshot_download and torch.load(weights_only=False)) and midtrain/gemma_moe.py are NOT shipped and fail closed with ImportError if reached; the author's run.sh (which installs packages from the network) was NEVER run. Instead we reproduce its settings (EFFORT=none, checkpoint 05901-c33): vLLM 0.30.0 --served-model-name wald-26b-a4b --max-model-len 131072 --gpu-memory-utilization 0.90 --max-num-seqs 128 --max-logprobs 64 --seed 0 --language-model-only, and the author's reader python3 -m eval.systemone_vllm --vllm http://127.0.0.1:8371 --gate 0 --budget 512 --wide knockout --template chat --prompt-format plain --max-model-len 131072 --noul-decisive-floor 0.81 --host 127.0.0.1 --port 8000. Image: official vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 plus two hash-pinned wheels (transformers 5.17.0, huggingface-hub 1.33.0) built offline on the VM (image sha256:dc93fedeea119e31b524c2352f397326d3b315cf129d33c0a7001cfce7c8e56e); torch 2.13.0+cu130, tokenizers 0.23.2. DISCLOSED DEVIATIONS from run.sh: run.sh not executed; reader bound to 127.0.0.1 (run.sh uses 0.0.0.0); DO_NOT_TRACK and HF_HUB_DISABLE_TELEMETRY added; the source-pins check is replaced by sha256 verification of all 10 files before upload. The submitted code was reviewed before the run, with no blocking findings (no download, subprocess, env read, exec/eval, pickle, torch.load or telemetry in the 10 shipped modules). The source and runner were reviewed; findings concerning the model card and measurement-window guard were resolved in the preregistration amendments before the run. Weights: org2ai/Wald-26B-A4B at 4a5d0b94bff26a8bba359ef311b3e535cefae901 (Apache-2.0, public, not gated, 39 files, 11 safetensors shards totalling 51,612,008,604 bytes (51,644,629,065 bytes for all 39 files), no pickle; Gemma-4-26B-A4B-it after full-parameter decision training, base google/gemma-4-26B-A4B-it@4d7ae4984b7db7de8f8457170b3f1a419ee76d52), sha256-verified file by file against the Hub LFS oids (MODEL_MANIFEST.json sha256 cfbc591cc731191f99f0585f334a2735b5e984deb6b7808cf9a8357d89dcbe5e). FIRST MEASUREMENT (found by our own intake; Author submission, a submission by the authors; same family as the published wald-4b-v2-1 row, a different base). AUTHOR'S OWN READER, on our hardware, not the author's run.sh (see code_revision). BENCHMARK-AWARE OUTPUT POLICY (author-shipped, measured as shipped): the reader runs with --noul-decisive-floor 0.81 (eval/systemone_vllm.py:36-42, 488-502, and serving.json). It moves a Noul probability inside (0.19, 0.81) to the nearer edge; the author's own comment says 'JevBench v1.5/v1.6 (O1S) scores 1-f<P<f as an abstention'. It is a pure function of the model's own probability, with no item ids or item strings. EFFECT ON THIS RUN: 123 of 375 Noul answers (32.8 %) sit exactly on 0.81 or 0.19, no answer lies inside (0.19, 0.81), so the abstention rate is 0.0; the Noul calibration is measured as shipped and is not comparable to an unfloored run (the unfloored probabilities are not recorded). This is a disclosure per exposure ruling on file. AUTHOR EXPOSURE (self-reported, not verified): the README claims 219/231 on the JevBench public set (Jev 1.13: 200/231), run by the authors; training on public items is possible and was not checked by us (no overlap check; no JevBench use or denial declared). 10 s RULE: max measured decision latency 9.79 s (median 0.043 s, p95 0.058 s, 99th percentile 8.63 s); 0 decisions over 10 s, but the slowest decisions are close to the limit (in the smoke a 78.5k-token request took 8.8 s cold and 0.18 s warm). DETERMINISM: first 50 positions run twice, only 1/50 bit-identical, 0 argmax flips, max probability difference 0.042 (disclosed; the cause was not investigated). USAGE OFFSET: server usage.input_tokens = our offline count + 1 on every item; see cost_basis. CONTEXT: the author's limit is 131,072 tokens per read and a longer read is refused with HTTP 422, never truncated; the longest sealed item is 81,725 tokens, so nothing was refused; the smoke accepted 131,070 tokens. The scored configuration is EFFORT=none (no thinking); no sealed item had more than 26 options, so the knockout mode was never used. PRICING: the frozen module has no explicit listing decision for the card's exact base spelling; the estimate follows the published sibling rows (see cost_basis). No outside request for this model.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 62.28668771673929,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100.0,
        "score": 3.526729816884533
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 62.28668771673929
      },
      "calibration_gen": {
       "accuracy": 0.7128712871287128,
       "cc": 61.329080094640695,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 60.00000000000001,
        "score": 52.61926573461137
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.329080094640695
      },
      "composite_gen": {
       "accuracy": 0.8910891089108911,
       "cc": 79.88818031266844,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 85.18518518518519,
        "score": 64.23545331379572
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 79.88818031266844
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 80.95766193104647,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 60.00000000000001,
        "score": 82.87298579313939
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 80.95766193104647
      },
      "jaggedness": {
       "accuracy": 0.6888888888888889,
       "cc": 49.006962191036905,
       "cc_by_type": {
        "choice": 72.56977863330125,
        "noul": 40.54054054054055,
        "score": 33.910567399268906
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.006962191036905
      },
      "llm_router": {
       "accuracy": 0.8242811501597445,
       "cc": 71.5864116735259,
       "cc_by_type": {
        "choice": 73.84872765695167,
        "noul": 83.50515463917525,
        "score": 57.40535272445078
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.5864116735259
      },
      "long_state": {
       "accuracy": 0.6,
       "cc": -7.025166452017032,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": -100.0,
        "score": 17.63417806330375
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 40.61069467435372,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 75.0,
        "score": -3.6833798944646334
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 40.61069467435372
      },
      "multilingual": {
       "accuracy": 0.7527272727272727,
       "cc": 61.11914207752587,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": 58.33333333333333,
        "score": 51.43426570654784
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 61.11914207752587
      },
      "negation_pair_gen": {
       "accuracy": 0.8255813953488372,
       "cc": 59.75609874358286,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 53.33333333333334,
        "score": 32.60162956408189
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 59.75609874358286
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 80.35665685016336,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100.0,
        "score": 68.0969975775171
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.35665685016336
      },
      "usecase": {
       "accuracy": 0.7,
       "cc": 54.9190856886155,
       "cc_by_type": {
        "choice": 68.1372549019608,
        "noul": 50.0,
        "score": 46.62000216388572
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.9190856886155
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 29.362565407181737,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100.0,
        "score": -54.76944663559764
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 29.362565407181737
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 16.94109176007718,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 4.893935802504967
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 16.94109176007718
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 72.45376024931926,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75.0,
        "score": 66.25508605769228
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.45376024931926
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.55632535362075,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.52186246682126
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.55632535362075
      },
      "en": {
       "accuracy": 0.7819287576020851,
       "cc": 64.4227407367809,
       "cc_by_type": {
        "choice": 77.15429394320303,
        "noul": 64.02877697841727,
        "score": 52.0851512887224
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 64.4227407367809
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 65.13915722074792,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100.0,
        "score": 5.205399884103468
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 65.13915722074792
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.65429624620543,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 99.20332589708623
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.65429624620543
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 74.9477771440457,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100.0,
        "score": 67.19627260860767
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 74.9477771440457
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 57.73566501094746,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 75.0,
        "score": 31.726548105468066
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 57.73566501094746
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": -100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0.0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.337669469601714,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": -4.957373508575991
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.337669469601714
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 36.332437701064066,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 11.888496582143748
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.332437701064066
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 59.15338662842026,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0,
        "score": 21.904604329705222
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 59.15338662842026
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.60356597056011,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.81069791168032
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.60356597056011
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 15.363590673359726,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 5.398131327438471
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 15.363590673359726
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 73.71413499154939,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 68.75984043804215
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 73.71413499154939
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 67.58621040803877,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100.0,
        "score": 55.79339846041565
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 67.58621040803877
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 56.70325456902729,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 61.173593494315924
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 56.70325456902729
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 51.526643189721035,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100.0,
        "score": -23.680939996054274
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 51.526643189721035
      },
      "uk": {
       "accuracy": 1.0,
       "cc": 99.06371815169098,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 97.19115445507296
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 99.06371815169098
      },
      "zh": {
       "accuracy": 1.0,
       "cc": 76.19757397633792,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 28.592721929013752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 76.19757397633792
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8146666666666667,
       "cc": 76.06153963910522,
       "cc_by_type": {
        "choice": 76.06153963910522
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 76.06153963910522
      },
      "noul": {
       "accuracy": 0.824,
       "cc": 64.8,
       "cc_by_type": {
        "noul": 64.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 64.8
      },
      "score": {
       "accuracy": 0.6453333333333333,
       "cc": 50.36096329833563,
       "cc_by_type": {
        "score": 50.36096329833563
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.36096329833563
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 18.962327091129648,
       "cc_by_type": {
        "choice": 20.0,
        "noul": 50.0,
        "score": -13.11301872661106
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 18.962327091129648
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 71.39826677963767,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 14.194800338912984
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 71.39826677963767
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 44.39453904633217,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50.0,
        "score": 15.703942342248522
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.39453904633217
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 11.335982228981843,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -29.14994804989657
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 11.335982228981843
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 46.06728146848554,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 19.361264695311707
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 46.06728146848554
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": -25.58952390664666,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -218.27800568220414
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 50.44808275364195,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 56.666377112466456
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 50.44808275364195
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 51.727112859171505,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 50.0,
        "score": 46.993185267409984
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 51.727112859171505
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 30.020847503119555,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 32.562542509358664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 30.020847503119555
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 40.73034293084367,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 40.2495584142298
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 40.73034293084367
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 57.44797224134985,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 50.638490367460435
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 57.44797224134985
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 42.85782288237548,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100.0,
        "score": 25.347662195513553
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 42.85782288237548
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 57.469447177637996,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 63.924656312184624
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 57.469447177637996
      },
      "model_routing": {
       "accuracy": 0.8525345622119815,
       "cc": 74.45948584990684,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 96.96969696969697,
        "score": 46.2788904501534
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.45948584990684
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 82.11215077126832,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100.0,
        "score": 69.3690818723462
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 82.11215077126832
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 57.71735862375096,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 56.531834459925136
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.71735862375096
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 53.55195584334256,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 100.0,
        "score": 19.769168022638517
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 53.55195584334256
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 33.766528904412525,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0.0,
        "score": 44.777847582802785
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 33.766528904412525
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 86.18780733194313,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100.0,
        "score": 79.91573516308917
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 86.18780733194313
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.6478419735516,
      88.60393644006602
     ],
     "I": [
      60.90990743631795,
      69.0211072229648
     ],
     "capability": [
      72.50042821110986,
      78.13972271554701
     ],
     "composite": {
      "A": [
       66.33206507388527,
       68.99172007480622
      ],
      "B": [
       64.97009367669243,
       68.7071684124402
      ],
      "C": [
       66.33206507388527,
       68.99172007480622
      ]
     },
     "seed": 86
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.05577650422971,
     "capability": 76.78216129185128,
     "intelligence": 65.50854607947286
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8146666666666667,
      "calibration": 92.70989473534586,
      "cc": 76.06153963910522,
      "cc_by_type": {
       "choice": 76.06153963910522
      },
      "cc_tiered": 74.41605361563016,
      "cc_tiered_open": 73.59023844561248,
      "cc_tiered_sealed": 75.24186878564784,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.06153963910522
     },
     "noul": {
      "accuracy": 0.824,
      "calibration": 80.37852722292858,
      "cc": 64.8,
      "cc_by_type": {
       "noul": 64.8
      },
      "cc_tiered": 69.86991870936465,
      "cc_tiered_open": 83.56521739130434,
      "cc_tiered_sealed": 56.17462002742497,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.8
     },
     "score": {
      "accuracy": 0.6453333333333333,
      "calibration": 91.07890755441471,
      "cc": 50.36096329833563,
      "cc_by_type": {
       "score": 50.36096329833563
      },
      "cc_tiered": 52.239665913423764,
      "cc_tiered_open": 57.34460946093709,
      "cc_tiered_sealed": 47.134722365910434,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.36096329833563
     }
    },
    "raw_on_A": {
     "calibration": 84.82817901188946,
     "capability": 73.65390073325852,
     "intelligence": 62.47962245462759
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.705914979984016,
    "Emphasis on Accuracy 60:20:20": 63.986001295101325,
    "Emphasis on Cost 20:20:60": 56.741885657480076,
    "Emphasis on Speed 20:60:20": 72.25796409560672,
    "Intelligence only": 64.41077922536263
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "quevirus (Hugging Face)",
   "axes": {
    "calibration": 85.13454850231766,
    "cost": 65.96610922261206,
    "intelligence": 52.14523729900746,
    "speed": 91.64762937997935
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06366853885427957,
      "ece_score": 87.26629222914409,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 87.26629222914409,
      "score_before_time_limit": 87.26629222914409,
      "time_limit_failed_share": 0.0
     },
     "noul": {
      "abstention_rate": 0.0,
      "brier": 0.15445726927323383,
      "ece": 0.10011666645915095,
      "n": 373,
      "score": 79.9766667081698,
      "score_before_time_limit": 79.9766667081698,
      "time_limit_failed_share": 0.0
     },
     "score": {
      "interval90_coverage": 0.946236559139785,
      "interval90_width": 0.4881272401433691,
      "n": 372,
      "nrps": 0.10731253403237839,
      "score": 88.1606865696391,
      "score_before_time_limit": 88.1606865696391,
      "time_limit_failed_share": 0.0,
      "top_ece": 0.06473686728741979
     }
    },
    "score": 85.13454850231766
   },
   "capability": 68.63989290066256,
   "composite_ci95": {
    "A": [
     58.46527025828537,
     72.03101892867859
    ],
    "B": [
     53.74101119496074,
     68.15919800485098
    ],
    "C": [
     40.600882123809285,
     62.93143660148717
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference through the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026): base string Qwen/Qwen3.5-4B priced as Qwen/Qwen3.5-4B USD 0.03/M input, USD 0.15/M output. The module selects the deprecated-after-cutoff DeepInfra reference retained under the frozen policy (price snapshots of 25 Sep 2026), USD 0.03/M input and USD 0.15/M output. Estimated, not charged, not a quote from the author or any host. Server-reported usage is supplied by our labelled first-party shim: exact native token_ids length summed over questions, with output_tokens=0; the native engine generates no text. Reported usage is 674,634 input tokens and 0 output tokens, matching the evaluator's offline count for the 1,485 answered items, offset 0 and 0 item-level mismatches. The 15 attempted long items fail with HTTP 500 CUDA OOM and report no usage; they contribute 0 reported tokens, not their 1,179,066 offline prompt tokens. Those are the same standard long items excluded from every row's common 1,485-item cost basis (v1.6.30), so common-basis usage is also 674,634 input tokens and 0 output tokens. Measured arithmetic: USD 0.013628970 per 1,000 decisions on the common basis = 0.210992220x the unchanged USD 0.06459465517241379 cap; separately USD 0.013492680/1,000 over all 1,500 items = 0.208882298x.  COST ON THE COMMON BASIS: USD 0.013629 per 1,000 decisions on 1,485 items, 0.211 times the official cost cap USD 0.064595; estimated, not charged.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01349268
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013628969696969697
   },
   "decision_limit": {
    "limit_s": 10.0,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Xevn-4B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner. Native engine and first-party HTTP shim in a --network none --read-only container; serial requests, read-only code/model/input, blank credential variables and HF offline flags. In-container cut proof before sealed input: only loopback, no default routes, DNS and TCP fail; model inventory, author and shim hashes checked. Measured 24 Oct 2026 09:20-09:26Z, about USD 0.29 rental; measurement container 126 s, run cap 1592 s, complete=true, deadline_hit=false, rc 0; scored loop 81.776480 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 2526d8bfedc67e8a680615052231a6c32fa62aa800b2e31cbe0aaad7afe1001b. Row cleanup and final node shred verified; exact-id VM/volume teardown confirms provider absent and guard closed at 09:26:37Z. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 57.189441526420964,
    "I_sealed": 47.10103307159395,
    "base": 52.14523729900746,
    "excess": 0.2488816844713,
    "gap": 10.088408454827011,
    "penalty": 1.0,
    "per_type_split": {
     "open|choice": {
      "cc": 65.60765951512785,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 62.25228059138094,
       "judge": 72.31377719182598,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 69.6425120772947,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100.0,
       "hard": 65.21739130434783,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 36.31815298684035,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.361958198992475,
       "hard": 31.632819097025468,
       "judge": 39.75557897140969,
       "standard": 32.51077918354005
      }
     },
     "sealed|choice": {
      "cc": 60.25280499391855,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 82.84625158831004,
       "hard": 53.16294668671215,
       "judge": 55.92556317335946,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 53.389344572051726,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 44.827586206896555,
       "hard": 54.54545454545454,
       "judge": 41.53846153846155,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 27.66094964881158,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 47.45316925731022,
       "hard": 19.995217233285057,
       "judge": 25.622754060555874,
       "standard": 36.15359805799887
      }
     }
    }
   },
   "jevbench_score": 70.18285597413652,
   "key": "xevn-4b",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card and LICENSE, recorded by the release owner); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "quevirus/Xevn-4B@ba11f0c2d8100a1e8b7789a1174ddee7c24dcab0",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.7935656836461126,
    "decisive": 373,
    "decisive_rate": 1.0,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1.0,
   "rank": 27,
   "ranked": true,
   "ranks": {
    "A": 27,
    "B": 44,
    "C": 52,
    "capability": 102
   },
   "repo": "https://huggingface.co/quevirus/Xevn-4B",
   "scores": {
    "A": 70.18285597413652,
    "B": 65.64162224825803,
    "C": 53.01000347219501
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23218741199993928,
    "p50_s_raw": 0.04109370599996964,
    "p95_s_adjusted": 0.29471238780007525,
    "p95_s_raw": 0.07235619390003763
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHOR'S OWN INFERENCE, run unmodified: quevirus/Xevn-4B at ba11f0c2d8100a1e8b7789a1174ddee7c24dcab0, sole 139-line author file xevn_infer.py (sha256 3c0e434c03e7cc96675afb86b5a9cb5f76f3af07d20f64035329600512f4a267), wrapped by our first-party shim_xevn.py for HTTP, usage and per-item error recovery. Native Xevn('/model', dtype='bfloat16', device='cuda'), eager text forward, use_cache=False, FP32 final-position hidden state and selected tied-head readout. No generation, chat template, quantization or SDPA substitution. Native calibration preserved: choice temperature 1.1, noul and score 1.2, score_lambda=0; when 0.2<P(yes)<0.8, noul snaps to 0.2 below 0.5 and to 0.8 otherwise (tie 0.5 -> 0.8). Score is the native expected ordinal. Torch 2.13.0, Transformers 5.18.0, tokenizers 0.23.2, safetensors 0.8.0, huggingface-hub 1.33.0; optional fla/causal_conv1d/kernels excluded. vLLM 0.30.0 is the image substrate, not the serving backend. Base image vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90; derived image sha256:bebbb6b15cdef220e99bca031ae8b13a0c48a622e4bfcb01afe48ca07da6f0e5. The submitted code was reviewed and cleared before the run, together with evaluator exposure, the device and native inference without an input cap, as preregistered. The runner was reviewed, findings fixed, and the final changes and hashes rechecked before the run. Source review records Hub-capable from_pretrained and conditional hf_hub_download paths; local /model with temps.json, exact inventory/hashes, offline flags and network-none avoid runtime downloads. No author shell/subprocess, eval/exec, pickle or trust_remote_code path in the reviewed serving code. Weights: Apache-2.0, public, ungated; metadata 4,659,865,088 parameters, two BF16 safetensors shards totalling 9,319,828,160 bytes.  FIRST MEASUREMENT of Xevn-4B (our own intake). AUTHOR'S OWN INFERENCE on our rented hardware, one request at a time; first-party HTTP/usage shim preserves the native prompt, probabilities, temperature calibration and noul 0.2/0.8 confidence hardening. NATIVE OUTPUTS: choice argmax and probabilities, noul P(yes), score expected ordinal and probabilities. AUTHOR EXPOSURE: intake records author-reported English/Korean suite figures, not a comparable Decision Index Full or JevBench draw result; no overlap or contamination audit was run by us. The author's reported figures are not independently verified by us. CONTEXT: native, NO input cap, no evaluator-imposed cap, no truncation, splitting or fallback (ruling on file). The 16,384-token long-item boundary is a preregistration/fit classification, not a serving limit. All 1,500 items were ATTEMPTED. NOT ANSWERED, counted wrong: exactly the 15 pre-registered long positions in the pre-registered long-item set failed with HTTP 500 cuda_out_of_memory; native BF16 eager attention cannot hold these approximately 76k-80.5k-token prompts on the 96 GB card. One BF16 attention score tensor alone is approximately 186-215 GB at the 76,309-82,000-token planning/synthetic bounds. The shim frees CUDA memory after each failed item and keeps serving; no author inference change. This is the pre-registered class B complete outcome, with no failure outside the long set. Answered 1,485 of 1,500 items HTTP 200; longest normal item 1,789 native tokens. 10 s RULE: 0 decisions over 10 s; the 15 OOM items already count wrong as failures, and no additional decision is counted wrong by the time rule in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 1.070494 s, median 0.041094 s, p95 0.072361 s over all 1,500 requests; OOM items fail in about 1 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. Pre-sealed synthetic probes: 16,384 tokens answered HTTP 200; 76,309 and 82,000 tokens attempted and failed HTTP 500, followed each time by healthy service, a successful short request and GPU memory back to about 9.5 GB. ",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 72.73849014174947,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100.0,
        "score": 59.882137091915055
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 72.73849014174947
      },
      "calibration_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 44.81310011472409,
       "cc_by_type": {
        "choice": 58.642629904559925,
        "noul": 60.00000000000001,
        "score": 15.796670439612326
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 44.81310011472409
      },
      "composite_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 29.505656511603615,
       "cc_by_type": {
        "choice": 60.975609756097555,
        "noul": 11.111111111111116,
        "score": 16.430248667602175
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 29.505656511603615
      },
      "fairness": {
       "accuracy": 1.0,
       "cc": 95.36247478442279,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 86.08742435326837
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 95.36247478442279
      },
      "jaggedness": {
       "accuracy": 0.5111111111111111,
       "cc": 24.650295817093042,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": 35.13513513513513,
        "score": -3.4364132276481163
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 24.650295817093042
      },
      "llm_router": {
       "accuracy": 0.7891373801916933,
       "cc": 68.7514533359268,
       "cc_by_type": {
        "choice": 67.50902527075812,
        "noul": 87.62886597938144,
        "score": 51.11646875764083
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.7514533359268
      },
      "long_state": {
       "accuracy": 0.0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100.0,
        "score": -300.0
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0.0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": 9.847004603183725,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25.0,
        "score": 28.767817933262517
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 9.847004603183725
      },
      "multilingual": {
       "accuracy": 0.6945454545454546,
       "cc": 54.31575643740647,
       "cc_by_type": {
        "choice": 66.7427453537659,
        "noul": 55.55555555555556,
        "score": 40.64896840289794
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 54.31575643740647
      },
      "negation_pair_gen": {
       "accuracy": 0.5813953488372093,
       "cc": 15.718276984577555,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": 13.33333333333333,
        "score": -26.178502379600666
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 15.718276984577555
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 78.66172310053767,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100.0,
        "score": 60.009193325637035
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.66172310053767
      },
      "usecase": {
       "accuracy": 0.6181818181818182,
       "cc": 52.426873394583446,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": 66.66666666666667,
        "score": 34.73160057590717
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.426873394583446
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 25.048133307805642,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50.0,
        "score": 1.3348761138931131
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 25.048133307805642
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100.0
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 51.83174838150152,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 33.33333333333333,
        "score": 22.161911811171255
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 51.83174838150152
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 71.69130854287623,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75.0,
        "score": 56.00312916845173
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 71.69130854287623
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 69.67338519389119,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 66.66666666666667,
        "score": 92.63281852394545
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 69.67338519389119
      },
      "en": {
       "accuracy": 0.6681146828844483,
       "cc": 49.101428893202375,
       "cc_by_type": {
        "choice": 61.33803590388204,
        "noul": 58.273381294964025,
        "score": 27.69286948076105
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 49.101428893202375
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 62.81669649155109,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100.0,
        "score": 8.0259459183727
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.81669649155109
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 28.814955843993364,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 51.472189936351675
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 28.814955843993364
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 37.98131330045231,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0.0,
        "score": 56.2968810778275
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 37.98131330045231
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 37.043463968559664,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25.0,
        "score": 19.64994497830469
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 37.043463968559664
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100.0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 76.72593226780768,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100.0,
        "score": 44.20741488604191
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 76.72593226780768
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 55.77291194289271,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 80.0,
        "score": 44.50698046728021
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 55.77291194289271
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 35.36568109185147,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 0.0,
        "score": 6.097043275554414
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 35.36568109185147
      },
      "mixed": {
       "accuracy": 1.0,
       "cc": 99.4107347631596,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 100.0,
        "score": 98.23220428947874
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.4107347631596
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 25.24092775397595,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 61.00416854331312
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 25.24092775397595
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75.0,
       "cc_by_type": {
        "choice": 100.0,
        "noul": 50.0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75.0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 44.26397834155194,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 28.0268059514438
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 44.26397834155194
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 72.00491020494218,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100.0,
        "score": 47.835590956606154
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.00491020494218
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 28.80327255884877,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": 60.00000000000001,
        "score": 28.537477251014387
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 28.80327255884877
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 35.571600287366806,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50.0,
        "score": 21.93219216644824
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 35.571600287366806
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 71.12345297189192,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100.0,
        "score": 45.62842343180479
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 71.12345297189192
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 27.727390622429468,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 0.6962961610737173
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 27.727390622429468
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.708,
       "cc": 62.2840084961442,
       "cc_by_type": {
        "choice": 62.2840084961442
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 62.2840084961442
      },
      "noul": {
       "accuracy": 0.7893333333333333,
       "cc": 57.86666666666667,
       "cc_by_type": {
        "noul": 57.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 57.86666666666667
      },
      "score": {
       "accuracy": 0.49333333333333335,
       "cc": 30.459370980009403,
       "cc_by_type": {
        "score": 30.459370980009403
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 30.459370980009403
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 31.992860243105383,
       "cc_by_type": {
        "choice": 52.0,
        "noul": 25.0,
        "score": 18.978580729316143
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 31.992860243105383
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 33.778547910211756,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 33.33333333333333,
        "score": 5.269391142643542
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 33.778547910211756
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 51.26446710355204,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100.0,
        "score": 2.573889115534189
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.26446710355204
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 40.84074431959357,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 100.0,
        "score": 16.08948442076902
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 40.84074431959357
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 32.54651784203781,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 59.95839410582355
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 32.54651784203781
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": -11.142280648793653,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -141.60294257531177
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 56.96253848985584,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 71.42857142857142,
        "score": 19.06688717825101
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.96253848985584
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -0.9722016882584134,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -50.0,
        "score": 30.707088315015696
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 33.296585222788686,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 60.00000000000001,
        "score": 44.05642233503272
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 33.296585222788686
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 33.718022386517426,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 66.66666666666667,
        "score": 24.423160664191823
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.718022386517426
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 59.1340259848485,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 77.77777777777777,
        "score": 22.88011413025609
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 59.1340259848485
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 23.608514769162785,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 2.0083400064130896
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 23.608514769162785
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 62.99586745712046,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 100.0,
        "score": 46.56917626771456
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 62.99586745712046
      },
      "model_routing": {
       "accuracy": 0.7926267281105991,
       "cc": 67.82357865219124,
       "cc_by_type": {
        "choice": 74.28571428571429,
        "noul": 90.90909090909092,
        "score": 38.27593076176851
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.82357865219124
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 73.52337478888262,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100.0,
        "score": 43.602753925189106
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 73.52337478888262
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 73.07772917610153,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100.0,
        "score": 52.65937137231573
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 73.07772917610153
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.7863946384018,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0.0,
        "score": 12.694159284663531
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.7863946384018
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 39.57707246355063,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 28.876144926883775
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 39.57707246355063
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 85.60432469497714,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100.0,
        "score": 67.48913066856133
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.60432469497714
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.67264371922077,
      87.00306451828784
     ],
     "I": [
      46.73117169541663,
      56.15596364566082
     ],
     "capability": [
      63.86721895874797,
      71.11242130154535
     ],
     "composite": {
      "A": [
       58.46527025828537,
       72.03101892867859
      ],
      "B": [
       53.74101119496074,
       68.15919800485098
      ],
      "C": [
       40.600882123809285,
       62.93143660148717
      ]
     },
     "seed": 85
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.13454850231766,
     "capability": 68.63989290066256,
     "intelligence": 52.14523729900746
    },
    "per_type": {
     "choice": {
      "accuracy": 0.708,
      "calibration": 87.26629222914409,
      "cc": 62.2840084961442,
      "cc_by_type": {
       "choice": 62.2840084961442
      },
      "cc_tiered": 62.930232254523204,
      "cc_tiered_open": 65.60765951512785,
      "cc_tiered_sealed": 60.25280499391855,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 62.2840084961442
     },
     "noul": {
      "accuracy": 0.7893333333333333,
      "calibration": 79.9766667081698,
      "cc": 57.86666666666667,
      "cc_by_type": {
       "noul": 57.86666666666667
      },
      "cc_tiered": 61.51592832467321,
      "cc_tiered_open": 69.6425120772947,
      "cc_tiered_sealed": 53.389344572051726,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 57.86666666666667
     },
     "score": {
      "accuracy": 0.49333333333333335,
      "calibration": 88.1606865696391,
      "cc": 30.459370980009403,
      "cc_by_type": {
       "score": 30.459370980009403
      },
      "cc_tiered": 31.989551317825967,
      "cc_tiered_open": 36.31815298684035,
      "cc_tiered_sealed": 27.66094964881158,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.459370980009403
     }
    },
    "raw_on_A": {
     "calibration": 85.60181885840427,
     "capability": 69.85120932695071,
     "intelligence": 54.10059979549715
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.30147345400096,
    "Emphasis on Accuracy 60:20:20": 59.80698361987809,
    "Emphasis on Cost 20:20:60": 66.16691940295598,
    "Emphasis on Speed 20:60:20": 74.54833719186517,
    "Intelligence only": 52.14523729900746
   }
  }
 ],
 "tier_weights": {
  "easy": 0.1,
  "hard": 0.4,
  "judge": 0.3,
  "standard": 0.2
 },
 "types": {
  "choice": 0.3333333333333333,
  "noul": 0.3333333333333333,
  "score": 0.3333333333333333
 },
 "v16": {
  "a2_supplement": {
   "counts": {
    "A": 300,
    "P": 300,
    "S": 300,
    "api_input": 600,
    "api_smoke_public": 5,
    "core": 0,
    "selfhosted_input": 600
   },
   "release": "v1.6.0-A2",
   "set_manifest_sha256": {
    "A": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d",
    "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba",
    "S": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d"
   },
   "supplement_of": "v1.6.0"
  },
  "capability_formula": "mean(Intelligence, Calibration)",
  "counts": {
   "A": 300,
   "P": 300,
   "S": 1200,
   "api_input": 1500,
   "api_smoke_public": 5,
   "core": 900,
   "selfhosted_input": 1500
  },
  "equating": {
   "equating_flag": null,
   "min_pool": 5,
   "offsets": {
    "C": 2.3428887908654588,
    "I": -2.209917383979878
   },
   "pool_note": "Historical v1.6.0 equating offsets, kept for the record. Hosted APIs that answered the full set (v1.6.1) are not equated.",
   "rule": "median over complete full-coverage self-hosted systems of metric(S u P) - metric(A u P), per metric (Intelligence, Calibration); API value = clip(raw_A + offset)"
  },
  "equating_A2": {
   "G_med_A2_basis_P_vs_A2": 3.824932047119341,
   "diagnostic_main_A_offset_same_systems": {
    "C": 1.9486117388517954,
    "I": -4.381056886943014
   },
   "equating_flag": null,
   "offset_bootstrap_B": 200,
   "offsets": {
    "C": 1.5638258868659562,
    "I": -0.2887166717454388
   },
   "per_system": {
    "hopper": {
     "C_A2": 87.82068552843725,
     "C_A_main": 87.40904942545944,
     "C_S": 90.07902805935959,
     "I_A2": 34.77120638990772,
     "I_A_main": 41.88389851544817,
     "I_S": 34.73970870443837,
     "dC": 2.2583425309223344,
     "dI": -0.03149768546934695
    },
    "kev-0.6b": {
     "C_A2": 68.46161035185293,
     "C_A_main": 65.17717796738081,
     "C_S": 66.26655558235618,
     "I_A2": 5.866380901403119,
     "I_A_main": 9.100409843273326,
     "I_S": 7.598789169415934,
     "dC": -2.1950547694967497,
     "dI": 1.732408268012815
    },
    "kev-4b": {
     "C_A2": 68.65918994957973,
     "C_A_main": 69.97372951951687,
     "C_S": 68.45285936403559,
     "I_A2": 19.558821195537895,
     "I_A_main": 25.023085194237723,
     "I_S": 19.510707720813777,
     "dC": -0.20633058554413708,
     "dI": -0.04811347472411853
    },
    "kev-8b": {
     "C_A2": 61.3782880436933,
     "C_A_main": 67.24332350856244,
     "C_S": 70.7561729087212,
     "I_A2": 26.42642690493986,
     "I_A_main": 34.2874494947909,
     "I_S": 29.26217941280341,
     "dC": 9.377884865027895,
     "dI": 2.8357525078635497
    },
    "localjev-qwen3.5-4b": {
     "C_A2": 76.77849725935185,
     "C_A_main": 81.40355743693019,
     "C_S": 85.84286362718517,
     "I_A2": 22.674863142962646,
     "I_A_main": 28.26930138473988,
     "I_S": 24.145363218466134,
     "dC": 9.064366367833316,
     "dI": 1.4705000755034874
    },
    "malkuth-4b": {
     "C_A2": 83.46678991361165,
     "C_A_main": 85.35554104202156,
     "C_S": 88.43741591513763,
     "I_A2": 34.73305384360792,
     "I_A_main": 38.819192380023466,
     "I_S": 33.840703805987175,
     "dC": 4.970626001525986,
     "dI": -0.8923500376207443
    },
    "metask-jev-4b": {
     "C_A2": 79.1415556283946,
     "C_A_main": 83.18850868556241,
     "C_S": 84.41575352936586,
     "I_A2": 41.051921128504205,
     "I_A_main": 41.86894313160032,
     "I_S": 39.166454612935595,
     "dC": 5.274197900971259,
     "dI": -1.8854665155686092
    },
    "raw-phi-4-mini": {
     "C_A2": 56.153985406303406,
     "C_A_main": 53.16092441580774,
     "C_S": 57.015465878675315,
     "I_A2": 14.189557846849354,
     "I_A_main": 12.927281214813327,
     "I_S": 13.660237978082595,
     "dC": 0.861480472371909,
     "dI": -0.5293198687667591
    },
    "raw-qwen3-1.7b": {
     "C_A2": 11.425912172792259,
     "C_A_main": 11.45520847967844,
     "C_S": 11.516326812547591,
     "I_A2": 4.994829708920566,
     "I_A_main": 6.997731901083261,
     "I_S": 5.124286554530066,
     "dC": 0.09041463975533226,
     "dI": 0.1294568456095
    },
    "raw-qwen3-4b-instruct-2507": {
     "C_A2": 34.31244267053921,
     "C_A_main": 37.75992930947443,
     "C_S": 35.18175191334879,
     "I_A2": 35.69189443401588,
     "I_A_main": 38.58935302861775,
     "I_S": 33.95117742100547,
     "dC": 0.8693092428095781,
     "dI": -1.7407170130104106
    },
    "raw-qwen3-8b": {
     "C_A2": 34.345565335170505,
     "C_A_main": 38.60246474366818,
     "C_S": 33.721548452619835,
     "I_A2": 30.34325924789806,
     "I_A_main": 31.73129272246627,
     "I_S": 26.54912920590792,
     "dC": -0.62401688255067,
     "dI": -3.7941300419901403
    },
    "typecastlm": {
     "C_A2": 76.627802179825,
     "C_A_main": 79.3986887571586,
     "C_S": 83.29951051808085,
     "I_A2": 31.896472425382363,
     "I_A_main": 31.998923946026117,
     "I_S": 29.78900656204624,
     "dC": 6.671708338255854,
     "dI": -2.107465863336124
    }
   },
   "pool": [
    "hopper",
    "kev-0.6b",
    "kev-4b",
    "kev-8b",
    "localjev-qwen3.5-4b",
    "malkuth-4b",
    "metask-jev-4b",
    "raw-phi-4-mini",
    "raw-qwen3-1.7b",
    "raw-qwen3-4b-instruct-2507",
    "raw-qwen3-8b",
    "typecastlm"
   ],
   "pool_n": 12,
   "rule": "median over the A2 pool of metric(S u P, main run) - metric(A2 u P, A2 run), per metric; API A2 value = clip(raw_A2 + offset_A2)",
   "runs_status": {
    "hopper": {
     "answered_ok": 600,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "kev-0.6b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "kev-4b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "kev-8b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "localjev-qwen3.5-4b": {
     "answered_ok": 600,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "malkuth-4b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "metask-jev-4b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-phi-4-mini": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-qwen3-1.7b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-qwen3-4b-instruct-2507": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-qwen3-8b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "typecastlm": {
     "answered_ok": 600,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    }
   }
  },
  "full_set_api_rows": [
   "fastino-gliner-2-5-decide",
   "jev-1.13.0",
   "sage-1.3.0",
   "wity-1",
   "wity-1-always",
   "wity-1-off"
  ],
  "g_med_rule": "median gap over ranked self-hosted systems; API systems use the same systems' P-vs-A gap",
  "item_sets": [
   {
    "answered_by": "self-hosted systems (run offline on our own GPU pods or Sandy) and, from v1.6.1, externally hosted APIs",
    "by_type": {
     "choice": 600,
     "noul": 300,
     "score": 300
    },
    "items": 1200,
    "name": "Sealed release draw",
    "set": "S"
   },
   {
    "answered_by": "every system (inside S since v1.6.1); retired for future draws",
    "by_type": {
     "choice": 150,
     "noul": 75,
     "score": 75
    },
    "items": 300,
    "name": "API subset (part of S; used by the v1.6.0 hosted-API measurements)",
    "set": "A"
   },
   {
    "answered_by": "every system",
    "by_type": {
     "choice": 150,
     "noul": 75,
     "score": 75
    },
    "items": 300,
    "name": "Public set",
    "set": "P"
   }
  ],
  "long_items_note": "23 items of this draw are very long (about 77,000 to 81,000 input tokens). Systems with a shorter context window refuse them; Jev-Omni runs out of GPU memory on them on its listed RTX 6000 (48 GB) recipe; decider-4b v2's server truncates them to about 32,800 tokens and answers; Plumb-4B reads them in full. Each outcome is the system's own recipe and is scored as such.",
  "offset_bootstrap_B": 200,
  "packaging_partial": false,
  "release": "v1.6.0",
  "scorer_sources_sha256": {
   "noul_method_v16.py": "119e59486ab3300b972fbcbadb0fe1dceceb124287607375aa61c66063c607d0",
   "score_v15.py": "0931e05c6410b1b971201834397cb609cdb25f942723c50375632952f6dcc447",
   "score_v15_headlineA.py": "826f2cbcb57007ec8a161e6eb0111182b8a656c0638027c6c78022a858fd8e2c",
   "score_v16.py": "02132271117653e6302c509232a581d0094f0e78eac8b1bfc67596fd5d5097e9"
  },
  "scoring_parameters": {
   "B": 1000,
   "g_med_fixed": null,
   "noul_method": "O1S",
   "only": null,
   "seed": 16,
   "test_allow_partial": false,
   "workers": 10
  },
  "set_manifest_sha256": {
   "A": "19b196a71d58da260cd83069ffcd64bb323e158c24a5fcabeffbb5bee7c1eabb",
   "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba",
   "S": "f13557a141fba12efc6bf6d39a0dd8e3bf803d4cacd6eab516d4f4f87045539a"
  }
 },
 "views": [
  "Balanced 33:33:33",
  "Emphasis on Accuracy 60:20:20",
  "Emphasis on Cost 20:20:60",
  "Emphasis on Speed 20:60:20",
  "Intelligence only"
 ]
}